From dca8911f240ea11ebb186c33573188aa9e1b031d Mon Sep 17 00:00:00 2001 From: "M.Shibuya" Date: Sat, 27 Feb 2016 15:03:30 +0900 Subject: [PATCH] Simplify export by trusting DB driver's encoding handling Refs. #2508 --- lib/rails_admin/adapters/active_record.rb | 11 +- lib/rails_admin/adapters/mongoid.rb | 2 +- lib/rails_admin/support/csv_converter.rb | 120 +----------------- spec/dummy_app/Gemfile | 28 ++-- spec/dummy_app/config/database.yml | 39 +++--- .../rails_admin/support/csv_converter_spec.rb | 48 ++++--- 6 files changed, 71 insertions(+), 177 deletions(-) diff --git a/lib/rails_admin/adapters/active_record.rb b/lib/rails_admin/adapters/active_record.rb index 099f682f1c..494fdc26c6 100644 --- a/lib/rails_admin/adapters/active_record.rb +++ b/lib/rails_admin/adapters/active_record.rb @@ -67,9 +67,14 @@ def properties delegate :primary_key, :table_name, to: :model, prefix: false def encoding - encoding = ::ActiveRecord::Base.connection.try(:encoding) - encoding ||= ::ActiveRecord::Base.connection.try(:charset) # mysql2 - encoding || 'UTF-8' + case ::ActiveRecord::Base.connection_config[:adapter] + when 'postgresql' + ::ActiveRecord::Base.connection.select_one("SELECT ''::text AS str;").values.first.encoding + when 'mysql2' + ::ActiveRecord::Base.connection.instance_variable_get(:@connection).encoding + else + ::ActiveRecord::Base.connection.select_one("SELECT '' AS str;").values.first.encoding + end end def embedded? diff --git a/lib/rails_admin/adapters/mongoid.rb b/lib/rails_admin/adapters/mongoid.rb index 485c80b53f..dee9147360 100644 --- a/lib/rails_admin/adapters/mongoid.rb +++ b/lib/rails_admin/adapters/mongoid.rb @@ -79,7 +79,7 @@ def table_name end def encoding - 'UTF-8' + Encoding::UTF_8 end def embedded? diff --git a/lib/rails_admin/support/csv_converter.rb b/lib/rails_admin/support/csv_converter.rb index 160ebdb235..1f71b49624 100644 --- a/lib/rails_admin/support/csv_converter.rb +++ b/lib/rails_admin/support/csv_converter.rb @@ -3,104 +3,6 @@ module RailsAdmin class CSVConverter - class DbEncodingMap - # The mapping from canonical encoding names in PostgreSQL to ones in Ruby. - # Taken from here: - # https://bitbucket.org/ged/ruby-pg/src/master/ext/pg.c - PG_ENCODINGS = { - 'BIG5' => Encoding::Big5, - 'EUC_CN' => Encoding::GB2312, - 'EUC_JP' => Encoding::EUC_JP, - 'EUC_JIS_2004' => Encoding::EUC_JP, - 'EUC_KR' => Encoding::EUC_KR, - 'EUC_TW' => Encoding::EUC_TW, - 'GB18030' => Encoding::GB18030, - 'GBK' => Encoding::GBK, - 'ISO_8859_5' => Encoding::ISO_8859_5, - 'ISO_8859_6' => Encoding::ISO_8859_6, - 'ISO_8859_7' => Encoding::ISO_8859_7, - 'ISO_8859_8' => Encoding::ISO_8859_8, - 'KOI8' => Encoding::KOI8_R, - 'KOI8R' => Encoding::KOI8_R, - 'KOI8U' => Encoding::KOI8_U, - 'LATIN1' => Encoding::ISO_8859_1, - 'LATIN2' => Encoding::ISO_8859_2, - 'LATIN3' => Encoding::ISO_8859_3, - 'LATIN4' => Encoding::ISO_8859_4, - 'LATIN5' => Encoding::ISO_8859_9, - 'LATIN6' => Encoding::ISO_8859_10, - 'LATIN7' => Encoding::ISO_8859_13, - 'LATIN8' => Encoding::ISO_8859_14, - 'LATIN9' => Encoding::ISO_8859_15, - 'LATIN10' => Encoding::ISO_8859_16, - 'MULE_INTERNAL' => Encoding::Emacs_Mule, - 'SJIS' => Encoding::Windows_31J, - 'SHIFT_JIS_2004' => Encoding::Windows_31J, - 'SQL_ASCII' => nil, - 'UHC' => Encoding::CP949, - 'UTF8' => Encoding::UTF_8, - 'WIN866' => Encoding::IBM866, - 'WIN874' => Encoding::Windows_874, - 'WIN1250' => Encoding::Windows_1250, - 'WIN1251' => Encoding::Windows_1251, - 'WIN1252' => Encoding::Windows_1252, - 'WIN1253' => Encoding::Windows_1253, - 'WIN1254' => Encoding::Windows_1254, - 'WIN1255' => Encoding::Windows_1255, - 'WIN1256' => Encoding::Windows_1256, - 'WIN1257' => Encoding::Windows_1257, - 'WIN1258' => Encoding::Windows_1258, - } - - # The mapping from canonical encoding names in MySQL to ones in Ruby. - # Taken from here: - # https://github.com/tmtm/ruby-mysql/blob/master/lib/mysql/charset.rb - # Author: TOMITA Masahiro - MYSQL_ENCODINGS = { - 'armscii8' => nil, - 'ascii' => Encoding::US_ASCII, - 'big5' => Encoding::Big5, - 'binary' => Encoding::ASCII_8BIT, - 'cp1250' => Encoding::Windows_1250, - 'cp1251' => Encoding::Windows_1251, - 'cp1256' => Encoding::Windows_1256, - 'cp1257' => Encoding::Windows_1257, - 'cp850' => Encoding::CP850, - 'cp852' => Encoding::CP852, - 'cp866' => Encoding::IBM866, - 'cp932' => Encoding::Windows_31J, - 'dec8' => nil, - 'eucjpms' => Encoding::EucJP_ms, - 'euckr' => Encoding::EUC_KR, - 'gb2312' => Encoding::EUC_CN, - 'gbk' => Encoding::GBK, - 'geostd8' => nil, - 'greek' => Encoding::ISO_8859_7, - 'hebrew' => Encoding::ISO_8859_8, - 'hp8' => nil, - 'keybcs2' => nil, - 'koi8r' => Encoding::KOI8_R, - 'koi8u' => Encoding::KOI8_U, - 'latin1' => Encoding::ISO_8859_1, - 'latin2' => Encoding::ISO_8859_2, - 'latin5' => Encoding::ISO_8859_9, - 'latin7' => Encoding::ISO_8859_13, - 'macce' => Encoding::MacCentEuro, - 'macroman' => Encoding::MacRoman, - 'sjis' => Encoding::SHIFT_JIS, - 'swe7' => nil, - 'tis620' => Encoding::TIS_620, - 'ucs2' => Encoding::UTF_16BE, - 'ujis' => Encoding::EucJP_ms, - 'utf8' => Encoding::UTF_8, - 'utf8mb4' => Encoding::UTF_8, - } - - def self.encodings - @_encodings ||= PG_ENCODINGS.merge MYSQL_ENCODINGS - end - end - def initialize(objects = [], schema = {}) return self if (@objects = objects).blank? @@ -130,29 +32,21 @@ def initialize(objects = [], schema = {}) end def to_csv(options = {}) - # encoding shenanigans first - encoding_from = DbEncodingMap.encodings[@abstract_model.encoding] || Encoding::UTF_8 - encoding_to = - if options[:encoding_to].present? - Encoding.find(options[:encoding_to]) - else - encoding_from - end + encoding_to = Encoding.find(options[:encoding_to]) if options[:encoding_to].present? csv_string = generate_csv_string(options) - - if encoding_to != encoding_from - csv_string = csv_string.encode(encoding_to, encoding_from, invalid: :replace, undef: :replace, replace: '?') + if encoding_to + csv_string = csv_string.encode(encoding_to, invalid: :replace, undef: :replace, replace: '?') end + # Add a BOM for utf8 encodings, helps with utf8 auto-detect for some versions of Excel. # Don't add if utf8 but user don't want to touch input encoding: # If user chooses utf8, they will open it in utf8 and BOM will disappear at reading. # But that way "English" users who don't bother and chooses to let utf8 by default won't get BOM added # and will not see it if Excel opens the file with a different encoding. - if options[:encoding_to].present? && encoding_to == Encoding::UTF_8 - csv_string = "\xEF\xBB\xBF#{csv_string}" - end - [!options[:skip_header], encoding_to.to_s, csv_string] + csv_string = "\xEF\xBB\xBF#{csv_string}" if encoding_to == Encoding::UTF_8 + + [!options[:skip_header], (encoding_to || csv_string.encoding).to_s, csv_string] end private diff --git a/spec/dummy_app/Gemfile b/spec/dummy_app/Gemfile index 04ba535f10..41230b49eb 100644 --- a/spec/dummy_app/Gemfile +++ b/spec/dummy_app/Gemfile @@ -5,28 +5,18 @@ gem 'rack-cache', require: 'rack/cache' group :active_record do platforms :jruby do - case ENV['CI_DB_ADAPTER'] - when 'mysql' - gem 'activerecord-jdbcmysql-adapter', '>= 1.2' - gem 'jdbc-mysql', '>= 5.1' - when 'postgresql' - gem 'activerecord-jdbcpostgresql-adapter', '>= 1.2' - gem 'jdbc-postgres', '>= 9.2' - else - gem 'activerecord-jdbcsqlite3-adapter', '>= 1.2' - gem 'jdbc-sqlite3', '>= 3.7' - end + gem 'activerecord-jdbcmysql-adapter', '>= 1.2' + gem 'jdbc-mysql', '>= 5.1' + gem 'activerecord-jdbcpostgresql-adapter', '>= 1.2' + gem 'jdbc-postgres', '>= 9.2' + gem 'activerecord-jdbcsqlite3-adapter', '>= 1.2' + gem 'jdbc-sqlite3', '>= 3.7' end platforms :ruby, :mswin, :mingw do - case ENV['CI_DB_ADAPTER'] - when 'mysql2' - gem 'mysql2', '~> 0.3.14' - when 'postgresql' - gem 'pg', '>= 0.14' - else - gem 'sqlite3', '>= 1.3' - end + gem 'mysql2', '~> 0.3.14' + gem 'pg', '>= 0.14' + gem 'sqlite3', '>= 1.3' end gem 'paper_trail', '~> 3.0' diff --git a/spec/dummy_app/config/database.yml b/spec/dummy_app/config/database.yml index 51a4dd459d..e5967ec396 100644 --- a/spec/dummy_app/config/database.yml +++ b/spec/dummy_app/config/database.yml @@ -1,25 +1,32 @@ -# SQLite version 3.x -# gem install sqlite3 -# -# Ensure the SQLite 3 gem is defined in your Gemfile -# gem 'sqlite3' -development: +sqlite: &sqlite adapter: sqlite3 - database: db/development.sqlite3 pool: 5 timeout: 5000 -# Warning: The database defined as "test" will be erased and -# re-generated from your development database when you run "rake". -# Do not set this db to the same as development or production. -test: - adapter: sqlite3 - database: db/test.sqlite3 +postgresql: + adapter: postgresql + database: rails_admin + username: postgres + password: + host: localhost + encoding: utf8 pool: 5 timeout: 5000 -production: - adapter: sqlite3 - database: db/production.sqlite3 +mysql: + adapter: mysql2 + database: rails_admin + username: root + password: + host: localhost + encoding: utf8 pool: 5 timeout: 5000 + +development: + <<: *sqlite + database: db/development.sqlite3 + +test: + <<: *sqlite + database: db/test.sqlite3 diff --git a/spec/rails_admin/support/csv_converter_spec.rb b/spec/rails_admin/support/csv_converter_spec.rb index 1668a5c68c..379c8a2ffe 100644 --- a/spec/rails_admin/support/csv_converter_spec.rb +++ b/spec/rails_admin/support/csv_converter_spec.rb @@ -32,35 +32,33 @@ subject { RailsAdmin::CSVConverter.new(objects, schema).to_csv(encoding_to: encoding) } - context 'when encoding FROM MySQL utf8mb4' do - let(:encoding) { 'UTF-8' } # default - - it 'exports to UTF-8 with BOM', active_record: true do - # MySQL connection may report its encoding as 'utf8mb4' - expect(::ActiveRecord::Base.connection).to receive(:encoding) { 'utf8mb4' } - expect(subject[1]).to eq 'UTF-8' - expect(subject[2].encoding).to eq Encoding::UTF_8 - expect(subject[2].unpack('H*').first). - to eq 'efbbbf4e756d6265722c4e616d650a312ce381aae381bee381880a' # have BOM - end - end - - context 'when encoding FROM MySQL latin1' do + context 'when encoding FROM latin1', active_record: true do let(:encoding) { '' } let(:objects) { FactoryGirl.create_list :player, 1, number: 1, name: 'Josè'.encode('ISO-8859-1') } - - it 'exports to ISO-8859-1', active_record: true do - # postgresql cannot load invalid data: invalid byte sequence for encoding "UTF8": 0xe8 - if ActiveRecord::Base.connection_config[:adapter] != 'postgresql' - expect(::ActiveRecord::Base.connection).to receive(:encoding) { 'latin1' } - expect(subject[1]).to eq 'ISO-8859-1' - expect(subject[2].encoding).to eq Encoding::ISO_8859_1 - expect(subject[2].unpack('H*').first). - to eq '4e756d6265722c4e616d650a312c4a6f73e80a' - else - expect(true).to eq(true) + before do + case ActiveRecord::Base.connection_config[:adapter] + when 'postgresql' + @connection = ActiveRecord::Base.connection.instance_variable_get(:@connection) + @connection.set_client_encoding('latin1') + when 'mysql2' + ActiveRecord::Base.connection.execute('SET NAMES latin1;') + end + end + after do + case ActiveRecord::Base.connection_config[:adapter] + when 'postgresql' + @connection.set_client_encoding('utf8') + when 'mysql2' + ActiveRecord::Base.connection.execute('SET NAMES utf8;') end end + + it 'exports to ISO-8859-1' do + expect(subject[1]).to eq 'ISO-8859-1' + expect(subject[2].encoding).to eq Encoding::ISO_8859_1 + expect(subject[2].unpack('H*').first). + to eq '4e756d6265722c4e616d650a312c4a6f73e80a' + end end context 'when encoding to UTF-8' do