Try to force UTF-8 encoding on CSV data and replace invalid and undef…

…ined characters with \uFFFD.
projectblacklight · Mar 5, 2015 · d31d83f · d31d83f
1 parent 67fbc7a
commit d31d83f
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 2 deletions.
diff --git a/app/controllers/spotlight/resources/upload_controller.rb b/app/controllers/spotlight/resources/upload_controller.rb
@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'csv'
 
 module Spotlight::Resources
@@ -29,7 +30,7 @@ def create
 
     def csv_upload
       file = csv_params[:url]
-      csv = CSV.parse(file.read, {headers:true, return_headers: false}).map(&:to_hash)
+      csv = CSV.parse(file.read, {headers:true, return_headers: false, encoding: 'utf-8'}).map(&:to_hash)
       Spotlight::AddUploadsFromCSV.perform_later(csv, current_exhibit, current_user)
       flash[:notice] = t('spotlight.resources.upload.csv.success', file_name: file.original_filename)
       redirect_to :back

diff --git a/app/jobs/spotlight/add_uploads_from_csv.rb b/app/jobs/spotlight/add_uploads_from_csv.rb
@@ -1,3 +1,4 @@
+# encoding: utf-8
 module Spotlight
   class AddUploadsFromCSV < ActiveJob::Base
     queue_as :default
@@ -8,7 +9,7 @@ class AddUploadsFromCSV < ActiveJob::Base
     end
 
     def perform(csv_data, exhibit, user)
-      csv_data.each do |row|
+      encoded_csv(csv_data).each do |row|
         if (url = row.delete("url")).present?
           Spotlight::Resources::Upload.create(
             remote_url_url: url,
@@ -19,5 +20,13 @@ def perform(csv_data, exhibit, user)
       end
 
     end
+    private
+    def encoded_csv(csv)
+      csv.map do |row|
+        row.map do |label, column|
+          [label, column.encode('UTF-8', invalid: :replace, undef: :replace, replace: "\uFFFD")] if column.present?
+        end.compact.to_h
+      end.compact
+    end
   end
 end
diff --git a/app/models/spotlight/resources/upload.rb b/app/models/spotlight/resources/upload.rb
@@ -1,3 +1,4 @@
+# encoding: utf-8
 module Spotlight
   class Resources::Upload < Spotlight::Resource
     mount_uploader :url, Spotlight::ItemUploader