From d31d83f6a1041748c526aca2b3c9948e02d96ae7 Mon Sep 17 00:00:00 2001
From: Jessie Keck <jessie.keck@gmail.com>
Date: Wed, 4 Mar 2015 18:12:56 -0800
Subject: [PATCH] Try to force UTF-8 encoding on CSV data and replace invalid
 and undefined characters with \uFFFD.

---
 .../spotlight/resources/upload_controller.rb          |  3 ++-
 app/jobs/spotlight/add_uploads_from_csv.rb            | 11 ++++++++++-
 app/models/spotlight/resources/upload.rb              |  1 +
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/app/controllers/spotlight/resources/upload_controller.rb b/app/controllers/spotlight/resources/upload_controller.rb
index fbe066503..32f092165 100644
--- a/app/controllers/spotlight/resources/upload_controller.rb
+++ b/app/controllers/spotlight/resources/upload_controller.rb
@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'csv'
 
 module Spotlight::Resources
@@ -29,7 +30,7 @@ def create
 
     def csv_upload
       file = csv_params[:url]
-      csv = CSV.parse(file.read, {headers:true, return_headers: false}).map(&:to_hash)
+      csv = CSV.parse(file.read, {headers:true, return_headers: false, encoding: 'utf-8'}).map(&:to_hash)
       Spotlight::AddUploadsFromCSV.perform_later(csv, current_exhibit, current_user)
       flash[:notice] = t('spotlight.resources.upload.csv.success', file_name: file.original_filename)
       redirect_to :back
diff --git a/app/jobs/spotlight/add_uploads_from_csv.rb b/app/jobs/spotlight/add_uploads_from_csv.rb
index 9a035e12c..0e0cdda25 100644
--- a/app/jobs/spotlight/add_uploads_from_csv.rb
+++ b/app/jobs/spotlight/add_uploads_from_csv.rb
@@ -1,3 +1,4 @@
+# encoding: utf-8
 module Spotlight
   class AddUploadsFromCSV < ActiveJob::Base
     queue_as :default
@@ -8,7 +9,7 @@ class AddUploadsFromCSV < ActiveJob::Base
     end
 
     def perform(csv_data, exhibit, user)
-      csv_data.each do |row|
+      encoded_csv(csv_data).each do |row|
         if (url = row.delete("url")).present?
           Spotlight::Resources::Upload.create(
             remote_url_url: url,
@@ -19,5 +20,13 @@ def perform(csv_data, exhibit, user)
       end
 
     end
+    private
+    def encoded_csv(csv)
+      csv.map do |row|
+        row.map do |label, column|
+          [label, column.encode('UTF-8', invalid: :replace, undef: :replace, replace: "\uFFFD")] if column.present?
+        end.compact.to_h
+      end.compact
+    end
   end
 end
diff --git a/app/models/spotlight/resources/upload.rb b/app/models/spotlight/resources/upload.rb
index a0cc793cd..e0e90d0ba 100644
--- a/app/models/spotlight/resources/upload.rb
+++ b/app/models/spotlight/resources/upload.rb
@@ -1,3 +1,4 @@
+# encoding: utf-8
 module Spotlight
   class Resources::Upload < Spotlight::Resource
     mount_uploader :url, Spotlight::ItemUploader