Merge pull request #350 from CLOSER-Cohorts/feature/276-three-and-fou…

…r-column-txt-import-exports

Feature/276 three and four column txt import exports

app/models/cc_question.rb

@@ -48,6 +48,7 @@ class CcQuestion < ::ControlConstruct
 
   delegate :label, to: :response_unit, allow_nil: true, prefix: true
   delegate :label, to: :question, allow_nil: true, prefix: :base
+  delegate :control_construct_scheme, to: :instrument, allow_nil: true
 
   def topic_conflict
     return unless topic

app/models/dataset.rb

@@ -82,4 +82,9 @@ def questions
   def qv_count
     self.qv_mappings.size
   end
+
+  def instance_name
+    return unless filename
+    filename.match(/(\S*).ddi32.rp.xml/)[1]
+  end
 end

app/models/dv_mapping.rb

@@ -13,4 +13,5 @@ class DvMapping < ReadOnlyRecord
 
   # Each DV mapping can only belong to one {Dataset}
   belongs_to :dataset
-end
+  delegate :instance_name, to: :dataset, allow_nil: true, prefix: true
+end

app/models/instrument.rb

@@ -367,6 +367,10 @@ def normalize_friendly_id(value)
     value.to_s.parameterize(preserve_case: true)
   end
 
+  def control_construct_scheme
+    prefix + '_ccs01'
+  end
+
   private
   # Creates an empty sequence as the top-sequence, i.e. parentless
   def add_top_sequence

app/models/qv_mapping.rb

@@ -17,10 +17,13 @@ class QvMapping < ReadOnlyRecord
   # Each QV mapping can only belong to one {Instrument}
   belongs_to :instrument
 
+  delegate :control_construct_scheme, to: :instrument, allow_nil: true
+  delegate :instance_name, to: :dataset, allow_nil: true, prefix: true
+
   # Return question reference with grid cell reference
   #
   # @return [String] Question reference
   def question_with_cell
     self.question + ((self.x.nil? || self.y.nil?) ? '' : "$#{self.x};#{self.y}")
   end
-end
+end

app/models/variable.rb

@@ -54,6 +54,8 @@ def <<(new_item)
   # Ensure that a variable does not have a conflicting topic
   validate :topic_conflict
 
+  delegate :instance_name, to: :dataset, prefix: true, allow_nil: true
+
   # Identify if variable is derived.
   #
   # @return [Boolean]

app/views/cc_questions/tq.txt.erb

@@ -1,3 +1,3 @@
 <% @collection.each do |qc| %>
-<%= qc.label %><%= "\t" %><%= qc.fully_resolved_topic_code %>
-<% end %>
+<%= qc.control_construct_scheme %><%= "\t" %><%= qc.label %><%= "\t" %><%= qc.fully_resolved_topic_code %>
+<% end %>

app/views/datasets/dv.txt.erb

@@ -1,3 +1,3 @@
 <% @object.dv_mappings.order(id: 'ASC').each do |map| %>
-<%= map.variable %><%= "\t" %><%= map.source %>
-<% end %>
+<%= map.dataset_instance_name %><%= "\t" %><%= map.variable %><%= "\t" %><%= map.dataset_instance_name %><%= "\t" %><%= map.source %>
+<% end %>

app/views/instruments/mapping.txt.erb

@@ -1,3 +1,3 @@
 <% @object.qv_mappings.order(id: 'ASC').each do |map| %>
-<%= map.question_with_cell %><%= "\t" %><%= map.variable %>
-<% end %>
+<%= map.control_construct_scheme %><%= "\t" %><%= map.question_with_cell %><%= "\t" %><%= map.dataset_instance_name %><%= "\t" %><%= map.variable %>
+<% end %>

app/views/variables/tv.txt.erb

@@ -1,3 +1,3 @@
 <% @collection.each do |v| %>
-<%= v.name %><%= "\t" %><%= v.fully_resolved_topic_code %>
-<% end %>
+<%= v.dataset_instance_name %><%= "\t" %><%= v.name %><%= "\t" %><%= v.fully_resolved_topic_code %>
+<% end %>

config/locales/en.yml

@@ -38,3 +38,18 @@ en:
           attributes:
             resolved_topic:
               variables_conflict: "conflict, you cannot assign variable '%{variable}' to '%{source}' because '%{variable}' has a topic of '%{variable_topic}'' while '%{source}' is using topic '%{source_topic}'"
+  importers:
+    txt:
+      mapper:
+        topic_v:
+          record_invalid_dataset: "Record Invalid as dataset name in file (%{dataset_from_line}) does not match the Dataset assigned to this import (%{dataset_from_object})."
+          wrong_number_of_columns: "Record Invalid you must have 3 columns (Dataset identifier, Variable name and Topic code). You only provided %{actual_number_of_columns}"
+        topic_v:
+          record_invalid_control_construct_scheme: "Record Invalid as Control Construct Scheme name in file (%{control_construct_scheme_from_line}) does not match the Control Construct Scheme assigned to this import (%{control_construct_scheme_from_object})."
+          wrong_number_of_columns: "Record Invalid you must have 3 columns (Control Construct Scheme, Question Construct, Dataset identifier and Variable name). You only provided %{actual_number_of_columns}"
+        mapping:
+          record_invalid_control_construct_scheme: "Record Invalid as Control Construct Scheme name in file (%{control_construct_scheme_from_line}) does not match the Control Construct Scheme assigned to this import (%{control_construct_scheme_from_object})."
+          wrong_number_of_columns: "Record Invalid you must have 4 columns (Control Construct Scheme, Question Construct, Dataset identifier and Variable name). You only provided %{actual_number_of_columns}"
+        dv:
+          record_invalid_dataset: "Record Invalid as dataset name in file (%{dataset_from_line}) does not match the Dataset assigned to this import (%{dataset_from_object})."
+          wrong_number_of_columns: "Record Invalid you must have 4 columns (Dataset identifier, Variable name, Dataset identifier and Source variable name). You only provided %{actual_number_of_columns}"

lib/importers/txt/mapper/dv.rb

@@ -3,8 +3,19 @@ def import(options = {})
     set_import_to_running
     vars = @object.variables.includes(:questions, :src_variables, :der_variables, :topic)
     begin
-        @doc.each do |v, s|
-          log :input, "#{v},#{s}"
+        @doc.each do |variable_dataset, v, source_dataset, s|
+          log :input, "#{variable_dataset},#{v},#{source_dataset},#{s}"
+          if variable_dataset.blank? || source_dataset.blank? || v.blank? || s.blank?
+            @errors = true
+            log :outcome, I18n.t('importers.txt.mapper.dv.wrong_number_of_columns', actual_number_of_columns: {a: source_dataset, b: variable_dataset, c: v, d: s}.compact.count)
+            write_to_log
+            next
+          elsif variable_dataset != @object.instance_name
+            @errors = true
+            log :outcome, I18n.t('importers.txt.mapper.dv.record_invalid_dataset', dataset_from_line: variable_dataset, dataset_from_object: @object.instance_name)
+            write_to_log
+            next
+          end
           var = vars.find_by_name v
           src = vars.find_by_name s
           log :matches, "matched to Variable (#{var}) AND Source (#{src})"

lib/importers/txt/mapper/mapping.rb

@@ -7,8 +7,21 @@ def initialize(thing, options)
   def import(options = {})
     Map.where(id: @object.maps.pluck(:id)).delete_all
     set_import_to_running
-    @doc.each do |q, v|
-      log :input, "#{q},#{v} "
+    @doc.each do |control_construct_scheme, q, dataset, v|
+      log :input, "#{control_construct_scheme},#{q},#{dataset},#{v}"
+
+      if control_construct_scheme.blank? || dataset.blank? || q.blank? || v.blank?
+        @errors = true
+        log :outcome, I18n.t('importers.txt.mapper.mapping.wrong_number_of_columns', actual_number_of_columns: {a: control_construct_scheme, b: q, c: v, d: dataset}.compact.count)
+        write_to_log
+        next
+      elsif control_construct_scheme != @object.control_construct_scheme
+        @errors = true
+        log :outcome, I18n.t('importers.txt.mapper.mapping.record_invalid_control_construct_scheme', control_construct_scheme_from_line: control_construct_scheme, control_construct_scheme_from_object: @object.control_construct_scheme)
+        write_to_log
+        next
+      end
+
       q_ident, q_coords = *q.split('$')
       qc = @object.cc_questions.find_by_label q_ident
 
@@ -19,6 +32,7 @@ def import(options = {})
         end
         return nil
       end
+
       var = multidimensional_variable_finder.call(v)
       log :matches, "matched to QuestionContruct(#{qc}) AND Variable (#{var})"
       if qc.nil? || var.nil?

lib/importers/txt/mapper/topic_q.rb

@@ -2,8 +2,19 @@ class Importers::TXT::Mapper::TopicQ < Importers::TXT::Mapper::Instrument
   def import(options = {})
     cc_question_ids_to_delete = @object.cc_questions.pluck(:id)
     set_import_to_running
-    @doc.each do |q, t|
-      log :input, "#{q},#{t}"
+    @doc.each do |control_construct_scheme, q, t|
+      log :input, "#{control_construct_scheme},#{q},#{t}"
+      if control_construct_scheme.blank? || q.blank? || t.blank?
+        @errors = true
+        log :outcome, I18n.t('importers.txt.mapper.topic_q.wrong_number_of_columns', actual_number_of_columns: {a: control_construct_scheme, b: q, c: t}.compact.count)
+        write_to_log
+        next
+      elsif control_construct_scheme != @object.control_construct_scheme
+        @errors = true
+        log :outcome, I18n.t('importers.txt.mapper.topic_q.record_invalid_control_construct_scheme', control_construct_scheme_from_line: control_construct_scheme, control_construct_scheme_from_object: @object.control_construct_scheme)
+        write_to_log
+        next
+      end
       qc = @object.cc_questions.find_by_label q
       topic = Topic.find_by_code t
       log :matches, "matched to QuestionContruct(#{qc}) AND Topic (#{topic})"

lib/importers/txt/mapper/topic_v.rb

@@ -3,8 +3,19 @@ def import(options = {})
     variable_ids_to_delete = @object.variables.pluck(:id)
     set_import_to_running
     vars = @object.variables.includes(:questions, :src_variables, :der_variables, :topic)
-    @doc.each do |v, t|
-      log :input, "#{v},#{t}"
+    @doc.each do |dataset, v, t|
+      log :input, "#{dataset},#{v},#{t}"
+      if dataset.blank? || v.blank? || t.blank?
+        @errors = true
+        log :outcome, I18n.t('importers.txt.mapper.topic_v.wrong_number_of_columns', actual_number_of_columns: {a: dataset, b: v, c: t}.compact.count)
+        write_to_log
+        next
+      elsif dataset != @object.instance_name
+        @errors = true
+        log :outcome, I18n.t('importers.txt.mapper.topic_v.record_invalid_dataset', dataset_from_line: dataset, dataset_from_object: @object.instance_name)
+        write_to_log
+        next
+      end
       if @doc.config[0]&.include? :icase
         var = vars.where('lower(name) = ?', v.downcase).first
       else

test/controllers/cc_questions_controller_test.rb

@@ -67,4 +67,15 @@ class CcQuestionsControllerTest < ActionController::TestCase
 
     assert_response :success
   end
+
+  test "should show tq.txt" do
+    instrument = FactoryBot.create(:instrument)
+    cc_question = FactoryBot.create(:cc_question, instrument: instrument, label: 'qc_abc_123')
+    topic = FactoryBot.create(:topic)
+    cc_question.topic = topic
+    get :tq, format: :txt, params: { instrument_id: instrument.id }
+    assert_response :success
+    parsed_response = response.body.split("\n").map{|line| line.split("\t")}
+    assert_equal parsed_response.first, [cc_question.control_construct_scheme, cc_question.label, cc_question.fully_resolved_topic_code]
+  end
 end

test/controllers/datasets_controller_test.rb

@@ -38,4 +38,16 @@ class DatasetsControllerTest < ActionController::TestCase
 
     assert_response :success
   end
+
+  test "should show dv.txt" do
+    dataset = FactoryBot.create(:dataset)
+    source_variable = FactoryBot.create(:variable, dataset: dataset)
+    derived_variable = FactoryBot.create(:variable, dataset: dataset)
+    derived_variable.src_variables << source_variable
+    get :dv, format: :txt, params: { id: dataset.id }
+    map = DvMapping.last
+    assert_response :success
+    parsed_response = response.body.split("\n").map{|line| line.split("\t")}
+    assert_equal parsed_response.first, [map.dataset_instance_name, derived_variable.name, map.dataset_instance_name,  source_variable.name]
+  end
 end

test/controllers/instruments_controller_test.rb

@@ -64,6 +64,19 @@ class InstrumentsControllerTest < ActionController::TestCase
     assert_response :success
   end
 
+  test "should show qv.txt" do
+    dataset = FactoryBot.create(:dataset)
+    variable = FactoryBot.create(:variable, dataset: dataset)
+    instrument = FactoryBot.create(:instrument)
+    cc_question = FactoryBot.create(:cc_question, instrument: instrument, label: 'qc_abc_123')
+    cc_question.maps.create(variable: variable)
+    get :mapping, format: :txt, params: { id: instrument.id }
+    map = QvMapping.where(question: cc_question.label).first
+    assert_response :success
+    parsed_response = response.body.split("\n").map{|line| line.split("\t")}
+    assert_equal parsed_response.first, [instrument.control_construct_scheme, map.question_with_cell, map.dataset_instance_name,  variable.name]
+  end
+
   # test "should destroy instrument" do
   #   puts Instrument.count
   #   assert_difference('Instrument.count', -1) do

test/controllers/variables_controller_test.rb

@@ -39,4 +39,15 @@ class VariablesControllerTest < ActionController::TestCase
 
     assert_response :success
   end
+
+  test "should show tv.txt" do
+    dataset = FactoryBot.create(:dataset)
+    variable = FactoryBot.create(:variable, dataset: dataset)
+    topic = FactoryBot.create(:topic)
+    variable.topic = topic
+    get :tv, format: :txt, params: { dataset_id: dataset.id }
+    assert_response :success
+    parsed_response = response.body.split("\n").map{|line| line.split("\t")}
+    assert_equal parsed_response.first, [dataset.instance_name, variable.name, variable.fully_resolved_topic_code]
+  end
 end

test/factories/datasets.rb

@@ -2,5 +2,6 @@
   factory :dataset do
     sequence(:name) { |n| "Dataset #{n}" }
     sequence(:study) { |n| "Study#{n}" }
+    sequence(:filename) { |n| "study_#{n}.ddi32.rp.xml" }
   end
 end

test/lib/importers/txt/mapper/dv_test.rb

@@ -0,0 +1,52 @@
+require 'test_helper'
+require 'active_support/core_ext/hash/conversions'
+
+class Importers::TXT::Mapper::DVTest < ActiveSupport::TestCase
+
+  setup do
+    @dataset = FactoryBot.create(:dataset)
+    @source_variable = FactoryBot.create(:variable, dataset: @dataset)
+    @derived_variable = FactoryBot.create(:variable, dataset: @dataset)
+    @derived_variable.src_variables << @source_variable
+  end
+
+  describe "where instrument already has dv mappings" do
+    describe "dv mappings from tab delimited text file" do
+      it "should create a new mappings for the source and derived variables" do
+        source_variable = FactoryBot.create(:variable, dataset: @dataset)
+        new_txt = "#{@dataset.instance_name}\t#{@derived_variable}\t#{@dataset.instance_name}\t#{source_variable.name}\n"
+        import = FactoryBot.create(:import, dataset: @dataset, import_type: 'ImportJob::DV')
+        doc = Document.create(file: new_txt, item: @dataset)
+        Importers::TXT::Mapper::DV.new(doc.id, {:object=>@dataset.id.to_s, :import_id => import.id}).import
+        import = import.reload
+        assert_equal('success', import.state)
+        last_dv_mapping = @dataset.reload.dv_mappings.last
+        assert_equal [source_variable.name, @derived_variable.name], [last_dv_mapping.source, last_dv_mapping.variable]
+      end
+    end
+  end
+  describe "dataset from the tab delimited text file does not match the instrument for the import" do
+    it "should mark the import has an error" do
+      other_dataset = FactoryBot.create(:dataset)
+      new_txt = "#{other_dataset.instance_name}\t#{@derived_variable}\t#{@dataset.instance_name}\t#{@source_variable.name}\n"
+      import = FactoryBot.create(:import, dataset: @dataset, import_type: 'ImportJob::DV')
+      doc = Document.create(file: new_txt, item: @dataset)
+      Importers::TXT::Mapper::DV.new(doc.id, {:object=>@dataset.id.to_s, :import_id => import.id}).import
+      import = import.reload
+      assert_equal('failure', import.state)
+      assert_equal(import.parsed_log.first[:outcome], I18n.t('importers.txt.mapper.dv.record_invalid_dataset', dataset_from_line: other_dataset.instance_name, dataset_from_object: @dataset.instance_name))
+    end
+  end
+  describe "tab delimited text contains less than 4 columns" do
+    it "should mark the import has an error" do
+      new_txt = "#{@dataset.instance_name}\t#{@derived_variable}\t#{@dataset.instance_name}\n#{@dataset.instance_name}\t#{@derived_variable}\n"
+      import = FactoryBot.create(:import, dataset: @dataset, import_type: 'ImportJob::DV')
+      doc = Document.create(file: new_txt, item: @dataset)
+      Importers::TXT::Mapper::DV.new(doc.id, {:object=>@dataset.id.to_s, :import_id => import.id}).import
+      import = import.reload
+      assert_equal('failure', import.state)
+      assert_equal(import.parsed_log.first[:outcome], I18n.t('importers.txt.mapper.dv.wrong_number_of_columns', actual_number_of_columns: 3))
+      assert_equal(import.parsed_log[1][:outcome], I18n.t('importers.txt.mapper.dv.wrong_number_of_columns', actual_number_of_columns: 2))
+    end
+  end
+end