Remove old parsing and related column

Author: orhantoy

app/jobs/parse_customer_import_job.rb

@@ -4,11 +4,7 @@ class ParseCustomerImportJob < ApplicationJob
   end
 
   def perform(customer_import)
-    if customer_import.parsed_header_at?
-      customer_import.parse_with_dynamic_headers!
-    else
-      customer_import.parse!
-    end
+    customer_import.parse_with_dynamic_headers!
   rescue => e
     customer_import.update(
       parsing_failed_at: Time.zone.now,

app/models/customer_import.rb

@@ -61,21 +61,6 @@ def can_be_finalized?
     started_finalizing_at.nil? && finalized_at.nil?
   end
 
-  def parse!
-    return if parsed_at?
-
-    csv_contents = uploaded_file.download
-    csv_contents.force_encoding("UTF-8")
-
-    transaction do
-      CSV.parse(csv_contents, headers: true) do |row_from_csv|
-        rows.create!(parsed_data: row_from_csv.to_hash)
-      end
-
-      touch(:parsed_at)
-    end
-  end
-
   def parse_header!
     return if parsed_header_at?
 

app/models/customer_import_row.rb

@@ -59,12 +59,8 @@ def as_customer_attributes
   private
 
   def read_value_from_column(column)
-    if owner.header_data
-      index = header_index_for(column)
-      index ? cell_data[index] : nil
-    else
-      parsed_data[column]
-    end
+    index = header_index_for(column)
+    index ? cell_data[index] : nil
   end
 
   def header_index_for(column)
@@ -82,13 +78,8 @@ def check_for_errors
 
   def check_for_duplicate_in_file
     if customer_name.present?
-      existing_row_with_same_customer_name =
-        if owner.header_data
-          index = header_index_for("customer_name")
-          index ? owner.rows.where("cell_data->>#{index} = ?", customer_name).first : nil
-        else
-          owner.rows.where("parsed_data->>'customer_name' = ?", customer_name).first
-        end
+      index_of_customer_name = header_index_for("customer_name")
+      existing_row_with_same_customer_name = owner.rows.where("cell_data->>#{index_of_customer_name} = ?", customer_name).first
 
       if existing_row_with_same_customer_name
         self.duplicated = true

db/migrate/20181208183038_delete_parsed_data.rb

@@ -0,0 +1,5 @@
+class DeleteParsedData < ActiveRecord::Migration[5.2]
+  def change
+    remove_column :customer_import_rows, :parsed_data
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2018_11_23_052441) do
+ActiveRecord::Schema.define(version: 2018_12_08_183038) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -40,7 +40,6 @@
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.bigint "owner_id", null: false
-    t.json "parsed_data"
     t.boolean "with_errors", default: false, null: false
     t.boolean "duplicated", default: false, null: false
     t.string "error_message"

test/controllers/customer_imports_controller_test.rb

@@ -33,11 +33,13 @@ class CustomerImportsControllerTest < ActionDispatch::IntegrationTest
     space = Space.create!(title: "Zeobix", slug: "ZBX")
     customer_import = space.customer_imports.new
 
+    csv_headers = ["customer_name", "address", "city", "country_name", "country_code"]
+
     begin
       csv_file = Tempfile.new(["upload", ".csv"])
 
       CSV.open(csv_file.path, "wb") do |csv|
-        csv << ["customer_name", "address", "city", "country_name", "country_code"]
+        csv << csv_headers
         csv << ["Shape A/S", "Njalsgade 17A", "Copenhagen", "", "dk"]
         csv << ["Devotus", "", "Hvidovre", "", "dk"]
       end
@@ -51,7 +53,13 @@ class CustomerImportsControllerTest < ActionDispatch::IntegrationTest
       csv_file.unlink
     end
 
-    customer_import.parse!
+    customer_import.header_data = {
+      "as_array" => csv_headers,
+      "index_mapping" => { "customer_name" => 0, "address" => 1, "city" => 2, "country_code" => 4 },
+    }
+    customer_import.save!
+
+    customer_import.parse_with_dynamic_headers!
 
     assert_difference "space.customers.count", 2 do
       perform_enqueued_jobs do

test/models/customer_import_test.rb

@@ -4,44 +4,6 @@
 class CustomerImportTest < ActiveSupport::TestCase
   test "parsing rows" do
     space = Space.create!(title: "Zeobix", slug: "ZBX")
-    space.customers.create!(name: "Farm Backup")
-
-    customer_import = space.customer_imports.new
-
-    begin
-      csv_file = Tempfile.new(["upload", ".csv"])
-
-      CSV.open(csv_file.path, "wb") do |csv|
-        csv << ["customer_name", "address", "city", "country_name", "country_code", "user_name", "user_email"]
-        csv << ["Shape A/S", "Njalsgade 17A", "København S", "", "dk", "Bacon", "bacon.the.dog@example.com"]
-        csv << ["Farm Backup", "Njalsgade 23C", "Copenhagen", "", "dk"]
-        csv << ["", "-", "-", "", "dk"]
-        csv << ["Duplicate Company", "", "", "", ""]
-        csv << ["Duplicate Company", "", "", "", ""]
-      end
-
-      csv_file.rewind
-
-      customer_import.uploaded_file.attach(io: csv_file, filename: "upload.csv")
-      customer_import.save!
-    ensure
-      csv_file.close
-      csv_file.unlink
-    end
-
-    customer_import.parse!
-
-    assert_equal 5, customer_import.rows.count
-    assert_equal 1, customer_import.rows_with_errors.count
-    assert_equal 2, customer_import.rows_with_duplicates.count
-    assert_equal 2, customer_import.valid_rows.count
-
-    assert_equal ["blank_name"], customer_import.rows_with_errors.pluck(:error_message)
-    assert_equal ["customer_already_exists", "duplicate_in_file"], customer_import.rows_with_duplicates.order(:id).pluck(:error_message)
-  end
-
-  test "parsing rows with dynamic columns" do
-    space = Space.create!(title: "Zeobix", slug: "ZBX")
 
     csv_headers = ["Kunde", "Adresse", "By", "Land", "Brugerens fulde navn", "Bruger-email"]
 
@@ -73,29 +35,28 @@ class CustomerImportTest < ActiveSupport::TestCase
 
   test "finalizing import" do
     space = Space.create!(title: "Zeobix", slug: "ZBX")
-    customer_import = space.customer_imports.new
 
-    begin
-      csv_file = Tempfile.new(["upload", ".csv"])
+    csv_headers = ["customer_name", "address", "city", "country_name", "country_code", "user_name", "user_email"]
 
-      CSV.open(csv_file.path, "wb") do |csv|
-        csv << ["customer_name", "address", "city", "country_name", "country_code", "user_name", "user_email"]
-        csv << ["Shape A/S", "Njalsgade 17A", "København S", "", "dk", "Bacon", "bacon.the.dog@example.com"]
-        csv << ["", "-", "-", "", "dk"]
-        csv << ["Duplicate Company", "", "", "", ""]
-        csv << ["Duplicate Company", "", "", "", ""]
-      end
+    csv_contents = ""
+    CSV.generate(csv_contents, headers: csv_headers, write_headers: true) do |csv|
+      csv << ["Shape A/S", "Njalsgade 17A", "København S", "", "dk", "Bacon", "bacon.the.dog@example.com"]
+      csv << ["", "-", "-", "", "dk"]
+      csv << ["Duplicate Company", "", "", "", ""]
+      csv << ["Duplicate Company", "", "", "", ""]
+    end
 
-      csv_file.rewind
+    csv_file = StringIO.new(csv_contents)
 
-      customer_import.uploaded_file.attach(io: csv_file, filename: "upload.csv")
-      customer_import.save!
-    ensure
-      csv_file.close
-      csv_file.unlink
-    end
+    customer_import = space.customer_imports.new
+    customer_import.uploaded_file.attach(io: csv_file, filename: "upload.csv")
+    customer_import.header_data = {
+      "as_array" => csv_headers,
+      "index_mapping" => { "customer_name" => 0, "address" => 1, "city" => 2, "country_code" => 4, "user_name" => 5, "user_email" => 6 },
+    }
+    customer_import.save!
 
-    customer_import.parse!
+    customer_import.parse_with_dynamic_headers!
 
     assert_difference -> { space.customers.count } => 2, -> { space.users.count } => 1 do
       customer_import.finalize!