diff --git a/Rakefile b/Rakefile index c774b1d..f439ee8 100644 --- a/Rakefile +++ b/Rakefile @@ -63,9 +63,11 @@ end begin require 'rcov/rcovtask' Rcov::RcovTask.new do |test| - test.libs << 'test' + test.libs << 'lib' + test.libs << 'test/lib' test.pattern = 'test/**/*test.rb' test.verbose = true + test.rcov_opts << "--exclude gems/*" end rescue LoadError task :rcov do diff --git a/lib/mysql2psql/config.rb b/lib/mysql2psql/config.rb index 6733f73..b77767a 100644 --- a/lib/mysql2psql/config.rb +++ b/lib/mysql2psql/config.rb @@ -27,7 +27,7 @@ def reset_configfile(filepath) file.close end - def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, force_truncate = false) + def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, supress_sequence_update = false, force_truncate = false, use_timezones = false) configtext = < @use_timezones}) end unless @supress_ddl _time2 = Time.now tables.each do |table| - writer.truncate(table) if force_truncate && supress_ddl + writer.truncate(table) if force_truncate && !supress_ddl writer.write_contents(table, reader) end unless @supress_data diff --git a/lib/mysql2psql/postgres_db_writer.rb b/lib/mysql2psql/postgres_db_writer.rb index 8265e1c..9e33d32 100644 --- a/lib/mysql2psql/postgres_db_writer.rb +++ b/lib/mysql2psql/postgres_db_writer.rb @@ -27,47 +27,55 @@ def open def close @conn.close end - + def exists?(relname) rc = @conn.exec("SELECT COUNT(*) FROM pg_class WHERE relname = '#{relname}'") (!rc.nil?) && (rc.to_a.length==1) && (rc.first.count.to_i==1) end - def write_table(table) + def write_sequence_update(table, options) + serial_key_column = table.columns.detect do |column| + column[:auto_increment] + end + + if serial_key_column + serial_key = serial_key_column[:name] + max_value = serial_key_column[:maxval].to_i < 1 ? 1 : serial_key_column[:maxval] + 1 + serial_key_seq = "#{table.name}_#{serial_key}_seq" + + if !options.supress_ddl + if @conn.server_version < 80200 + @conn.exec("DROP SEQUENCE #{serial_key_seq} CASCADE") if exists?(serial_key_seq) + else + @conn.exec("DROP SEQUENCE IF EXISTS #{serial_key_seq} CASCADE") + end + @conn.exec <<-EOF + CREATE SEQUENCE #{serial_key_seq} + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1 + EOF + end + + if !options.supress_sequence_update + puts "Updated sequence #{serial_key_seq} to current value of #{max_value}" + @conn.exec sqlfor_set_serial_sequence(table, serial_key_seq, max_value) + end + end + end + + def write_table(table, options) puts "Creating table #{table.name}..." primary_keys = [] - serial_key = nil - maxval = nil columns = table.columns.map do |column| - if column[:auto_increment] - serial_key = column[:name] - maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1 - end if column[:primary_key] primary_keys << column[:name] end - " " + column_description(column) + " " + column_description(column, options) end.join(",\n") - if serial_key - if @conn.server_version < 80200 - serial_key_seq = "#{table.name}_#{serial_key}_seq" - @conn.exec("DROP SEQUENCE #{serial_key_seq} CASCADE") if exists?(serial_key_seq) - else - @conn.exec("DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE") - end - @conn.exec <<-EOF - CREATE SEQUENCE #{table.name}_#{serial_key}_seq - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1 - EOF - - @conn.exec sqlfor_set_serial_sequence(table,serial_key,maxval) - end - if @conn.server_version < 80200 @conn.exec "DROP TABLE #{PGconn.quote_ident(table.name)} CASCADE;" if exists?(table.name) else diff --git a/lib/mysql2psql/postgres_file_writer.rb b/lib/mysql2psql/postgres_file_writer.rb index 5350c7d..78e6ed3 100644 --- a/lib/mysql2psql/postgres_file_writer.rb +++ b/lib/mysql2psql/postgres_file_writer.rb @@ -38,41 +38,53 @@ def truncate(table) end end - def write_table(table) - primary_keys = [] - serial_key = nil - maxval = nil - - columns = table.columns.map do |column| - if column[:auto_increment] - serial_key = column[:name] - maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1 - end - if column[:primary_key] - primary_keys << column[:name] - end - " " + column_description(column) - end.join(",\n") + def write_sequence_update(table, options) + serial_key_column = table.columns.detect do |column| + column[:auto_increment] + end - if serial_key + if serial_key_column + serial_key = serial_key_column[:name] + serial_key_seq = "#{table.name}_#{serial_key}_seq" + max_value = serial_key_column[:maxval].to_i < 1 ? 1 : serial_key_column[:maxval] + 1 @f << <<-EOF -- --- Name: #{table.name}_#{serial_key}_seq; Type: SEQUENCE; Schema: public +-- Name: #{serial_key_seq}; Type: SEQUENCE; Schema: public -- +EOF + + if !options.supress_ddl + @f << <<-EOF +DROP SEQUENCE IF EXISTS #{serial_key_seq} CASCADE; -DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE; - -CREATE SEQUENCE #{table.name}_#{serial_key}_seq +CREATE SEQUENCE #{serial_key_seq} INCREMENT BY 1 NO MAXVALUE NO MINVALUE CACHE 1; - -#{sqlfor_set_serial_sequence(table,serial_key,maxval)} - - EOF +EOF + end + + if !options.supress_sequence_update + @f << <<-EOF +#{sqlfor_set_serial_sequence(table, serial_key_seq, max_value)} +EOF + end end + end + + def write_table(table, options) + primary_keys = [] + serial_key = nil + maxval = nil + + columns = table.columns.map do |column| + if column[:primary_key] + primary_keys << column[:name] + end + " " + column_description(column, options) + end.join(",\n") @f << <<-EOF -- Table: #{table.name} diff --git a/lib/mysql2psql/postgres_writer.rb b/lib/mysql2psql/postgres_writer.rb index f347778..fae5e38 100644 --- a/lib/mysql2psql/postgres_writer.rb +++ b/lib/mysql2psql/postgres_writer.rb @@ -5,15 +5,11 @@ class Mysql2psql class PostgresWriter < Writer - def column_description(column) - "#{PGconn.quote_ident(column[:name])} #{column_type_info(column)}" + def column_description(column, options) + "#{PGconn.quote_ident(column[:name])} #{column_type_info(column, options)}" end - def column_type(column) - column_type_info(column).split(" ").first - end - - def column_type(column) + def column_type(column, options={}) if column[:auto_increment] 'integer' else @@ -29,9 +25,9 @@ def column_type(column) when 'decimal' "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})" when 'datetime', 'timestamp' - 'timestamp without time zone' + "timestamp with#{options[:use_timezones] ? '' : 'out'} time zone" when 'time' - 'time without time zone' + "time with#{options[:use_timezones] ? '' : 'out'} time zone" when 'tinyblob', 'mediumblob', 'longblob', 'blob', 'varbinary' 'bytea' when 'tinytext', 'mediumtext', 'longtext', 'text' @@ -97,8 +93,8 @@ def column_default(column) end end - def column_type_info(column) - type = column_type(column) + def column_type_info(column, options) + type = column_type(column, options) if type not_null = !column[:null] || column[:auto_increment] ? ' NOT NULL' : '' default = column[:default] || column[:auto_increment] ? " DEFAULT #{column_default(column)}" : '' @@ -131,7 +127,17 @@ def process_row(table, row) if column_type(column) == "bytea" row[index] = PGconn.escape_bytea(row[index]) else - row[index] = row[index].gsub(/\\/, '\\\\\\').gsub(/\n/,'\n').gsub(/\t/,'\t').gsub(/\r/,'\r') + if row[index] == '\N' || row[index] == '\.' + row[index] = '\\' + row[index] # Escape our two PostgreSQL-text-mode-special strings. + else + # Awesome side-effect producing conditional. Don't do this at home. + unless row[index].gsub!(/\0/, '').nil? + puts "Removed null bytes from string since PostgreSQL TEXT types don't allow the storage of null bytes." + end + + row[index] = row[index].dump + row[index] = row[index].slice(1, row[index].size-2) + end end elsif row[index].nil? # Note: '\N' not "\N" is correct here: @@ -145,11 +151,11 @@ def process_row(table, row) def truncate(table) end - def sqlfor_set_serial_sequence(table,serial_key,maxval) - "SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true);" + def sqlfor_set_serial_sequence(table, serial_key_seq, max_value) + "SELECT pg_catalog.setval('#{serial_key_seq}', #{max_value}, true);" end - def sqlfor_reset_serial_sequence(table,serial_key,maxval) - "SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{maxval}, true);" + def sqlfor_reset_serial_sequence(table, serial_key, max_value) + "SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{max_value}, true);" end end diff --git a/test/fixtures/seed_integration_tests.sql b/test/fixtures/seed_integration_tests.sql index 36f0c80..63123f4 100644 --- a/test/fixtures/seed_integration_tests.sql +++ b/test/fixtures/seed_integration_tests.sql @@ -31,6 +31,7 @@ INSERT INTO numeric_types_basics VALUES ( 5, 127, 255, 32767, 65535, 8388607, 16777215, 2147483647, 4294967295, 2147483647, 4294967295, 9223372036854775807, 18446744073709551615, 1, 1, 1, 1, 1, 1); + DROP TABLE IF EXISTS basic_autoincrement; CREATE TABLE basic_autoincrement ( auto_id INT(11) NOT NULL AUTO_INCREMENT, @@ -85,3 +86,34 @@ INSERT INTO test_boolean_conversion (test_name, tinyint_1) VALUES ('test-true-no CREATE OR REPLACE VIEW test_view AS SELECT b.test_name FROM test_boolean_conversion b; + +DROP TABLE IF EXISTS test_null_conversion; +CREATE TABLE test_null_conversion (column_a VARCHAR(10)); +INSERT INTO test_null_conversion (column_a) VALUES (NULL); + +DROP TABLE IF EXISTS test_datetime_conversion; +CREATE TABLE test_datetime_conversion ( + column_a DATETIME, + column_b TIMESTAMP, + column_c DATETIME DEFAULT '0000-00-00', + column_d DATETIME DEFAULT '0000-00-00 00:00', + column_e DATETIME DEFAULT '0000-00-00 00:00:00', + column_f TIME +); +INSERT INTO test_datetime_conversion (column_a, column_f) VALUES ('0000-00-00 00:00', '08:15:30'); + +DROP TABLE IF EXISTS test_index_conversion; +CREATE TABLE test_index_conversion (column_a VARCHAR(10)); +CREATE UNIQUE INDEX test_index_conversion ON test_index_conversion (column_a); + +DROP TABLE IF EXISTS test_foreign_keys_child; +DROP TABLE IF EXISTS test_foreign_keys_parent; +CREATE TABLE test_foreign_keys_parent (id INT NOT NULL, PRIMARY KEY (id)) ENGINE=INNODB; +CREATE TABLE test_foreign_keys_child (id INT, test_foreign_keys_parent_id INT, + INDEX par_ind (test_foreign_keys_parent_id), + FOREIGN KEY (test_foreign_keys_parent_id) REFERENCES test_foreign_keys_parent(id) ON DELETE CASCADE +) ENGINE=INNODB; + +DROP TABLE IF EXISTS test_enum; +CREATE TABLE test_enum (name ENUM('small', 'medium', 'large')); +INSERT INTO test_enum (name) VALUES ('medium'); \ No newline at end of file diff --git a/test/integration/convert_to_db_test.rb b/test/integration/convert_to_db_test.rb index 491beed..5faaa5f 100644 --- a/test/integration/convert_to_db_test.rb +++ b/test/integration/convert_to_db_test.rb @@ -5,15 +5,22 @@ class ConvertToDbTest < Test::Unit::TestCase def setup + $stdout = StringIO.new + $stderr = StringIO.new + seed_test_database @options=get_test_config_by_label(:localmysql_to_db_convert_all) @mysql2psql = Mysql2psql.new([@options.filepath]) @mysql2psql.convert @mysql2psql.writer.open end + def teardown @mysql2psql.writer.close delete_files_for_test_config(@options) + + $stdout = STDOUT + $stderr = STDERR end def exec_sql_on_psql(sql, parameters=nil) @@ -50,5 +57,76 @@ def test_boolean_conversion_of_null assert_nil null_record['bit_1'] assert_nil null_record['tinyint_1'] end - + + def test_null_conversion + result = exec_sql_on_psql('SELECT column_a FROM test_null_conversion').first + assert_nil result['column_a'] + end + + def test_datetime_conversion + result = exec_sql_on_psql('SELECT column_a, column_f FROM test_datetime_conversion').first + assert_equal '1970-01-01 00:00:00', result['column_a'] + assert_equal '08:15:30', result['column_f'] + end + + def test_datetime_defaults + result = exec_sql_on_psql(<<-SQL) + SELECT a.attname, + pg_catalog.format_type(a.atttypid, a.atttypmod), + (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid) for 128) + FROM pg_catalog.pg_attrdef d + WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) AS default + FROM pg_catalog.pg_attribute a + WHERE a.attrelid = 'test_datetime_conversion'::regclass AND a.attnum > 0 + SQL + + assert_equal 6, result.count + + result.each do |row| + if row["attname"] == "column_f" + assert_equal "time without time zone", row["format_type"] + else + assert_equal "timestamp without time zone", row["format_type"] + end + + case row["attname"] + when "column_a" + assert_nil row["default"] + when "column_b" + assert_equal "now()", row["default"] + when "column_c", "column_d", "column_e" + assert_equal "'1970-01-01 00:00:00'::timestamp without time zone", row["default"] + end + end + end + + def test_index_conversion + result = exec_sql_on_psql('SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = \'test_index_conversion\'::regclass').first + assert_equal "CREATE UNIQUE INDEX test_index_conversion_index ON test_index_conversion USING btree (column_a)", result["pg_get_indexdef"] + end + + def test_foreign_keys + result = exec_sql_on_psql("SELECT conname, pg_catalog.pg_get_constraintdef(r.oid, true) as condef FROM pg_catalog.pg_constraint r WHERE r.conrelid = 'test_foreign_keys_child'::regclass") + expected = {"condef" => "FOREIGN KEY (test_foreign_keys_parent_id) REFERENCES test_foreign_keys_parent(id)", "conname" => "test_foreign_keys_child_test_foreign_keys_parent_id_fkey"} + assert_equal expected, result.first + end + + def test_output + $stdout.rewind + actual = $stdout.read + + assert_match /Counting rows of test_foreign_keys_child/, actual + end + + def test_enum + result = exec_sql_on_psql(<<-SQL) + SELECT r.conname, pg_catalog.pg_get_constraintdef(r.oid, true) + FROM pg_catalog.pg_constraint r + WHERE r.conrelid = 'test_enum'::regclass AND r.contype = 'c' + ORDER BY 1 + SQL + + assert_equal 1, result.count + assert_equal "CHECK (name::text = ANY (ARRAY['small'::character varying, 'medium'::character varying, 'large'::character varying]::text[]))", result.first["pg_get_constraintdef"] + end end \ No newline at end of file diff --git a/test/integration/convert_to_file_test.rb b/test/integration/convert_to_file_test.rb index a9f91d3..2e77006 100644 --- a/test/integration/convert_to_file_test.rb +++ b/test/integration/convert_to_file_test.rb @@ -114,4 +114,8 @@ def test_autoincrement def test_should_not_copy_views_as_tables assert_no_match /CREATE TABLE "test_view"/, content end + + def test_truncate + assert_match /TRUNCATE/, content + end end \ No newline at end of file diff --git a/test/lib/test_helper.rb b/test/lib/test_helper.rb index 4b77519..79d7673 100644 --- a/test/lib/test_helper.rb +++ b/test/lib/test_helper.rb @@ -72,7 +72,7 @@ def get_test_config_by_label(name) when :localmysql_to_file_convert_nothing get_new_test_config(true, ['unobtainium'], ['kryptonite'], true, true, false) when :localmysql_to_file_convert_all - get_new_test_config(true, [], [], false, false, false) + get_new_test_config(true, [], [], false, false, true) when :localmysql_to_db_convert_all get_new_test_config(false, [], [], false, false, false) when :localmysql_to_db_convert_nothing