Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Set charset/collation properly for each text column if using MySQL. #414

Merged
merged 6 commits into from Aug 30, 2014
Merged

Set charset/collation properly for each text column if using MySQL. #414

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6d1caa4
Merge remote-tracking branch 'origin/dsander-omniauth'
knu Aug 20, 2014
db792cd
Set charset/collation properly for each text column if using MySQL.
knu Jul 25, 2014
59482ee
Add support for Ruby 1.9.x.
knu Aug 13, 2014
66d1ac0
Fall back utf8mb4 to utf8 if the server does not support it.
knu Aug 21, 2014
393e3c4
Follow the default collation of Rails rather than MySQL.
knu Aug 21, 2014
857b8ea
Merge remote-tracking branch 'origin/master' into set_charset_for_mysql
knu Aug 27, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion Gemfile.lock
View file
Open in desktop
Expand Up @@ -45,9 +45,10 @@ GEM
extlib (>= 0.9.15)
multi_json (>= 1.0.0)
bcrypt (3.1.7)
better_errors (1.1.0)
better_errors (2.0.0)
coderay (>= 1.0.0)
erubis (>= 2.6.6)
rack (>= 0.9.0)
binding_of_caller (0.7.2)
debug_inspector (>= 0.0.1)
bootstrap-kaminari-views (0.0.3)
Expand Down
3 changes: 3 additions & 0 deletions config/initializers/ar_mysql_column_charset.rb
View file
Open in desktop
@@ -0,0 +1,3 @@
ActiveSupport.on_load :active_record do
require 'ar_mysql_column_charset'
end
74 changes: 74 additions & 0 deletions db/migrate/20140813110107_set_charset_for_mysql.rb
View file
Open in desktop
@@ -0,0 +1,74 @@
class SetCharsetForMysql < ActiveRecord::Migration
def all_models
@all_models ||= [
Agent,
AgentLog,
Contact,
Event,
Link,
Scenario,
ScenarioMembership,
User,
UserCredential,
Delayed::Job,
]
end

def change
conn = ActiveRecord::Base.connection

# This is migration is for MySQL only.
return unless conn.is_a?(ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter)

reversible do |dir|
dir.up do
all_models.each { |model|
table_name = model.table_name

# `contacts` may not exist
next unless connection.table_exists? table_name

model.columns.each { |column|
name = column.name
type = column.type
limit = column.limit
options = {
limit: limit,
null: column.null,
default: column.default,
}

case type
when :string, :text
options.update(charset: 'utf8', collation: 'utf8_unicode_ci')
case name
when 'username'
options.update(limit: 767 / 4, charset: 'utf8mb4', collation: 'utf8mb4_unicode_ci')
when 'message', 'options', 'name', 'memory',
'handler', 'last_error', 'payload', 'description'
options.update(charset: 'utf8mb4', collation: 'utf8mb4_bin')
when 'type', 'schedule', 'mode', 'email',
'invitation_code', 'reset_password_token'
options.update(collation: 'utf8_bin')
when 'guid', 'encrypted_password'
options.update(charset: 'ascii', collation: 'ascii_bin')
end
else
next
end

change_column table_name, name, type, options
}

execute 'ALTER TABLE %s CHARACTER SET utf8 COLLATE utf8_unicode_ci' % table_name
}

execute 'ALTER DATABASE %s CHARACTER SET utf8 COLLATE utf8_unicode_ci' % conn.current_database
end

dir.down do
# Do nada; no use to go back
end
end
end
end
74 changes: 37 additions & 37 deletions db/schema.rb
View file
Open in desktop
Expand Up @@ -18,7 +18,7 @@

create_table "agent_logs", force: true do |t|
t.integer "agent_id", null: false
t.text "message", null: false
t.text "message", limit: 16777215, null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
t.integer "level", default: 3, null: false
t.integer "inbound_event_id"
t.integer "outbound_event_id"
Expand All @@ -28,24 +28,24 @@

create_table "agents", force: true do |t|
t.integer "user_id"
t.text "options"
t.string "type"
t.string "name"
t.string "schedule"
t.text "options", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin"
t.string "type", collation: "utf8_bin"
t.string "name", charset: "utf8mb4", collation: "utf8mb4_bin"
t.string "schedule", collation: "utf8_bin"
t.integer "events_count"
t.datetime "last_check_at"
t.datetime "last_receive_at"
t.integer "last_checked_event_id"
t.datetime "created_at"
t.datetime "updated_at"
t.text "memory"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.text "memory", limit: 2147483647, charset: "utf8mb4", collation: "utf8mb4_bin"
t.datetime "last_web_request_at"
t.integer "keep_events_for", default: 0, null: false
t.datetime "last_event_at"
t.datetime "last_error_log_at"
t.boolean "propagate_immediately", default: false, null: false
t.boolean "disabled", default: false, null: false
t.string "guid", null: false
t.string "guid", null: false, charset: "ascii", collation: "ascii_bin"
t.integer "service_id"
end

Expand All @@ -55,10 +55,10 @@
add_index "agents", ["user_id", "created_at"], name: "index_agents_on_user_id_and_created_at", using: :btree

create_table "delayed_jobs", force: true do |t|
t.integer "priority", default: 0
t.integer "attempts", default: 0
t.text "handler"
t.text "last_error"
t.integer "priority", default: 0
t.integer "attempts", default: 0
t.text "handler", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin"
t.text "last_error", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin"
t.datetime "run_at"
t.datetime "locked_at"
t.datetime "failed_at"
Expand All @@ -73,11 +73,11 @@
create_table "events", force: true do |t|
t.integer "user_id"
t.integer "agent_id"
t.decimal "lat", precision: 15, scale: 10
t.decimal "lng", precision: 15, scale: 10
t.text "payload"
t.datetime "created_at"
t.datetime "updated_at"
t.decimal "lat", precision: 15, scale: 10
t.decimal "lng", precision: 15, scale: 10
t.text "payload", limit: 2147483647, charset: "utf8mb4", collation: "utf8mb4_bin"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime "expires_at"
end

Expand Down Expand Up @@ -107,13 +107,13 @@
add_index "scenario_memberships", ["scenario_id"], name: "index_scenario_memberships_on_scenario_id", using: :btree

create_table "scenarios", force: true do |t|
t.string "name", null: false
t.integer "user_id", null: false
t.string "name", null: false, charset: "utf8mb4", collation: "utf8mb4_bin"
t.integer "user_id", null: false
t.datetime "created_at"
t.datetime "updated_at"
t.text "description"
t.boolean "public", default: false, null: false
t.string "guid", null: false
t.text "description", charset: "utf8mb4", collation: "utf8mb4_bin"
t.boolean "public", default: false, null: false
t.string "guid", null: false, charset: "ascii", collation: "ascii_bin"
t.string "source_url"
t.string "tag_bg_color"
t.string "tag_fg_color"
Expand Down Expand Up @@ -144,33 +144,33 @@
t.integer "user_id", null: false
t.string "credential_name", null: false
t.text "credential_value", null: false
t.datetime "created_at"
t.datetime "updated_at"
t.string "mode", default: "text", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "mode", default: "text", null: false, collation: "utf8_bin"
end

add_index "user_credentials", ["user_id", "credential_name"], name: "index_user_credentials_on_user_id_and_credential_name", unique: true, using: :btree

create_table "users", force: true do |t|
t.string "email", default: "", null: false
t.string "encrypted_password", default: "", null: false
t.string "reset_password_token"
t.string "email", default: "", null: false, collation: "utf8_bin"
t.string "encrypted_password", default: "", null: false, charset: "ascii", collation: "ascii_bin"
t.string "reset_password_token", collation: "utf8_bin"
t.datetime "reset_password_sent_at"
t.datetime "remember_created_at"
t.integer "sign_in_count", default: 0
t.integer "sign_in_count", default: 0
t.datetime "current_sign_in_at"
t.datetime "last_sign_in_at"
t.string "current_sign_in_ip"
t.string "last_sign_in_ip"
t.datetime "created_at"
t.datetime "updated_at"
t.boolean "admin", default: false, null: false
t.integer "failed_attempts", default: 0
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.boolean "admin", default: false, null: false
t.integer "failed_attempts", default: 0
t.string "unlock_token"
t.datetime "locked_at"
t.string "username", null: false
t.string "invitation_code", null: false
t.integer "scenario_count", default: 0, null: false
t.string "username", limit: 191, null: false, charset: "utf8mb4", collation: "utf8mb4_unicode_ci"
t.string "invitation_code", null: false, collation: "utf8_bin"
t.integer "scenario_count", default: 0, null: false
end

add_index "users", ["email"], name: "index_users_on_email", unique: true, using: :btree
Expand Down
110 changes: 110 additions & 0 deletions lib/ar_mysql_column_charset.rb
View file
Open in desktop
@@ -0,0 +1,110 @@
require 'active_record'

# Module#prepend support for Ruby 1.9
require 'prepend' unless Module.method_defined?(:prepend)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Worth adding some comments at the top of this explaining what it's for.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure!

module ActiveRecord::ConnectionAdapters
class ColumnDefinition
module CharsetSupport
attr_accessor :charset, :collation
end

prepend CharsetSupport
end

class TableDefinition
module CharsetSupport
def new_column_definition(name, type, options)
column = super
column.charset = options[:charset]
column.collation = options[:collation]
column
end
end

prepend CharsetSupport
end

class AbstractMysqlAdapter
module CharsetSupport
def prepare_column_options(column, types)
spec = super
conn = ActiveRecord::Base.connection
spec[:charset] = column.charset.inspect if column.charset && column.charset != conn.charset
spec[:collation] = column.collation.inspect if column.collation && column.collation != conn.collation
spec
end

def migration_keys
super + [:charset, :collation]
end

def utf8mb4_supported?
if @utf8mb4_supported.nil?
@utf8mb4_supported = !select("show character set like 'utf8mb4'").empty?
else
@utf8mb4_supported
end
end

def charset_collation(charset, collation)
[charset, collation].map { |name|
case name
when nil
nil
when /\A(utf8mb4(_\w*)?)\z/
if utf8mb4_supported?
$1
else
"utf8#{$2}"
end
else
name.to_s
end
}
end
end

prepend CharsetSupport

class SchemaCreation
module CharsetSupport
def column_options(o)
column_options = super
column_options[:charset] = o.charset unless o.charset.nil?
column_options[:collation] = o.collation unless o.collation.nil?
column_options
end

def add_column_options!(sql, options)
charset, collation = @conn.charset_collation(options[:charset], options[:collation])

if charset
sql << " CHARACTER SET #{charset}"
end

if collation
sql << " COLLATE #{collation}"
end

super
end
end

prepend CharsetSupport
end

class Column
module CharsetSupport
attr_reader :charset

def initialize(*args)
super
@charset = @collation[/\A[^_]+/] unless @collation.nil?
end
end

prepend CharsetSupport
end
end
end