Permalink
Browse files

Works with Postgres as export database

  • Loading branch information...
1 parent 44c07a6 commit 20e61585ea66222443de308203b1c38ece3bc863 Josh Graham and Tim Pope committed with delitescere Jun 22, 2010
View
@@ -146,8 +146,8 @@ if write_squeal
file.write <<-EOS
require 'squealer'
-import('localhost', 27017, 'development') # <--- Change this as needed
-export('localhost', 'root', '', 'export') # <--- Change this as needed
+import('mysql', 'localhost', 27017, 'development') # <--- Change this as needed
+export('mysql', 'localhost', 'root', '', 'export') # <--- Change this as needed
EOS
file.write(squeal)
@@ -4,7 +4,7 @@
import('localhost', 27017, 'development')
# connect to the target mysql database
-export('localhost', 'root', '', 'reporting_export')
+export('mysql', 'localhost', 'root', '', 'reporting_export')
# Here we extract, transform and load all documents in a collection...
@@ -2,6 +2,7 @@
require 'data_objects'
require 'mysql'
require 'do_mysql'
+require 'do_postgres'
require 'singleton'
@@ -14,8 +15,12 @@ def import_from(host, port, name)
@import_connection = Connection.new(@import_dbc)
end
- def export_to(host, username, password, name)
- @export_do = DataObjects::Connection.new("mysql://#{username}:#{password}@#{host}/#{name}")
+ def export_to(adapter, host, username, password, name)
+ @@all_export_connections ||= []
+ @export_do.dispose if @export_do
+
+ @export_do = DataObjects::Connection.new("#{adapter}://#{username}:#{password}@#{host}/#{name}")
+ @@all_export_connections << @export_do
end
def import
@@ -26,6 +31,10 @@ def export
@export_do
end
+ def upsertable?
+ @export_do.is_a? DataObjects::Mysql::Connection
+ end
+
class Connection
attr_reader :collections
@@ -71,5 +80,11 @@ def each
@progress_bar.finish if @progress_bar
end
end
+
+ private
+
+ def dispose_all_connections
+ @@all_export_connections.each {|c| c.dispose if c} if defined?(@@all_export_connections)
+ end
end
end
@@ -19,8 +19,7 @@ def initialize(database_connection, table_name, row_id=nil, &block)
@binding = block.binding
verify_table_name_in_scope
-
- @row_id = obtain_row_id(row_id)
+ @row_id = infer_row_id
@column_names = []
@column_values = []
@sql = ''
@@ -55,9 +54,12 @@ def obtain_row_id(row_id)
end
def infer_row_id
- (eval "#{@table_name}._id", @binding, __FILE__, __LINE__).to_s
+ (
+ (eval "#{@table_name}[:_id]", @binding, __FILE__, __LINE__) ||
+ (eval "#{@table_name}['_id']", @binding, __FILE__, __LINE__)
+ ).to_s
end
-
+3
def verify_table_name_in_scope
table = eval "#{@table_name}", @binding, __FILE__, __LINE__
raise ArgumentError, "The variable '#{@table_name}' is not a hashmap" unless table.is_a? Hash
@@ -84,11 +86,18 @@ def target
yield self
- @sql = "INSERT #{@table_name}"
- @sql << " (#{pk_name}#{column_names}) VALUES (?#{column_value_markers})"
- @sql << " ON DUPLICATE KEY UPDATE #{column_markers}"
+ insert_statement = %{INSERT INTO "#{@table_name}"}
+ insert_statement << %{ (#{pk_name}#{column_names}) VALUES ('#{@row_id}'#{column_value_markers})}
+ if Database.instance.upsertable?
+ insert_statement << %{ ON DUPLICATE KEY UPDATE #{column_markers}}
+ @sql = insert_statement
+ else
+ update_statement = %{UPDATE "#{@table_name}" SET #{column_markers} WHERE #{pk_name}='#{@row_id}'}
+ process_sql(update_statement)
+ @sql = update_statement + "; " + insert_statement
+ end
- execute_sql(@sql)
+ process_sql(insert_statement)
Queue.instance.pop
end
@@ -101,10 +110,16 @@ def targets
@@targets
end
- def execute_sql(sql)
- values = typecast_values * 2
- Database.instance.export.create_command(sql).execute_non_query(infer_row_id, *values)
- rescue Mysql::Error, TypeError
+ def process_sql(sql)
+ values = Database.instance.upsertable? ? typecast_values * 2 : typecast_values
+ execute_sql(sql, values)
+ end
+
+ def execute_sql(sql, values)
+ Database.instance.export.create_command(sql).execute_non_query(*values)
+ rescue DataObjects::IntegrityError
+ raise "Failed to execute statement: #{sql} with #{values.inspect}.\nOriginal Exception was: #{$!.to_s}" if Database.instance.upsertable?
+ rescue
raise "Failed to execute statement: #{sql} with #{values.inspect}.\nOriginal Exception was: #{$!.to_s}"
end
@@ -149,7 +164,7 @@ def typecast_values
end
def quote_identifier(name)
- "`#{name}`"
+ %{"#{name}"}
end
class Queue < DelegateClass(Array)
@@ -8,7 +8,7 @@
let(:databases) { Squealer::Database.instance }
def prepare_export_database
- databases.export_to('localhost', 'root', '', $db_name)
+ databases.export_to($db_adapter, 'localhost', $db_user, '', $db_name)
end
def squeal_basic_users_document(user=users_document)
@@ -41,8 +41,10 @@ def squeal_basic_users_document(user=users_document)
let :first_users_record do
dbc = databases.instance_variable_get('@export_do')
- reader = dbc.create_command('SELECT * FROM user').execute_reader
- reader.each { |x| break x }
+ reader = dbc.create_command(%{SELECT * FROM "user"}).execute_reader
+ result = reader.each { |x| break x }
+ reader.close
+ result
end
context "a new record" do
View
@@ -6,38 +6,31 @@
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
require 'squealer'
+require "spec_helper_dbms_#{ENV['EXPORT_DBMS']||'mysql'}"
Spec::Runner.configure do |config|
config.before(:suite) do
$db_name = "test_export_#{object_id}"
- create_test_db($db_name)
+ create_export_db($db_name)
+ create_import_db($db_name)
end
config.after(:suite) do
- drop_test_db($db_name)
- end
-
- def create_test_db(name)
- dbc = DataObjects::Connection.new("mysql://root@localhost/mysql")
- dbc.create_command("DROP DATABASE IF EXISTS #{name}").execute_non_query
- dbc.create_command("CREATE DATABASE #{name}").execute_non_query
- dbc.create_command("SET sql_mode='ANSI_QUOTES'").execute_non_query
-
- create_export_tables
+ Squealer::Database.instance.send(:dispose_all_connections)
+ drop_export_test_db($db_name)
- Squealer::Database.instance.import_from('localhost', 27017, $db_name)
- @mongo = Squealer::Database.instance.import.send(:instance_variable_get, '@dbc')
drop_mongo
- seed_import
end
- def drop_test_db(name)
- @my.close if @my
- dbc = DataObjects::Connection.new("mysql://root@localhost/mysql")
- dbc.create_command("DROP DATABASE IF EXISTS #{name}").execute_non_query
- dbc.close
+ config.after(:each) do
+ @export_dbc.dispose if @export_dbc
+ end
+ def create_import_db(name)
+ Squealer::Database.instance.import_from('localhost', 27017, name)
+ @mongo = Squealer::Database.instance.import.instance_variable_get('@dbc')
drop_mongo
+ seed_import
end
def drop_mongo
@@ -77,43 +70,9 @@ def seed_import
users.each { |user| @mongo.collection('users').save user }
end
- def create_export_tables
- command = <<-COMMAND.gsub(/\n\s*/, " ")
- CREATE TABLE "user" (
- "id" CHAR(24) NOT NULL ,
- "organization_id" CHAR(24) NOT NULL ,
- "name" VARCHAR(255) NULL ,
- "gender" CHAR(1) NULL ,
- "dob" DATETIME NULL ,
- "awesome" BOOLEAN NULL ,
- "fat" BOOLEAN NULL ,
- "symbolic" VARCHAR(255) NULL ,
- "interests" TEXT NULL ,
- PRIMARY KEY ("id") )
- COMMAND
- non_query(command)
-
- command = <<-COMMAND.gsub(/\n\s*/, " ")
- CREATE TABLE "activity" (
- "id" CHAR(24) NOT NULL ,
- "user_id" CHAR(24) NULL ,
- "name" VARCHAR(255) NULL ,
- "due_date" DATETIME NULL ,
- PRIMARY KEY ("id") )
- COMMAND
- non_query(command)
-
- command = <<-COMMAND.gsub(/\n\s*/, " ")
- CREATE TABLE "organization" (
- "id" CHAR(24) NOT NULL ,
- "disabled_date" DATETIME NULL ,
- PRIMARY KEY ("id") )
- COMMAND
- non_query(command)
- end
def truncate_export_tables
- non_query('DELETE FROM "user"')
+ non_query('TRUNCATE TABLE "user"')
non_query('TRUNCATE TABLE "activity"')
non_query('TRUNCATE TABLE "organization"')
end
@@ -123,10 +82,6 @@ def as_time(date)
end
def non_query(text)
- my.create_command(text).execute_non_query
- end
-
- def my
- @my ||= DataObjects::Connection.new("mysql://root@localhost/#{$db_name}")
+ export_dbc.create_command(text).execute_non_query
end
end
@@ -0,0 +1,56 @@
+def create_export_db(name)
+ $db_adapter = 'mysql'
+ dbc = DataObjects::Connection.new("mysql://root@localhost/mysql")
+ dbc.create_command("DROP DATABASE IF EXISTS #{name}").execute_non_query
+ dbc.create_command("CREATE DATABASE #{name}").execute_non_query
+ dbc.create_command("SET sql_mode='ANSI_QUOTES'").execute_non_query
+ create_export_tables
+ dbc.dispose
+end
+
+def drop_export_test_db(name)
+ @export_dbc.dispose if @export_dbc
+ dbc = DataObjects::Connection.new("mysql://root@localhost/mysql")
+ dbc.create_command("DROP DATABASE IF EXISTS #{name}").execute_non_query
+ dbc.dispose
+end
+
+def create_export_tables
+ command = <<-COMMAND.gsub(/\n\s*/, " ")
+ CREATE TABLE "user" (
+ "id" CHAR(24) NOT NULL ,
+ "organization_id" CHAR(24) NOT NULL ,
+ "name" VARCHAR(255) NULL ,
+ "gender" CHAR(1) NULL ,
+ "dob" DATETIME NULL ,
+ "awesome" BOOLEAN NULL ,
+ "fat" BOOLEAN NULL ,
+ "symbolic" VARCHAR(255) NULL ,
+ "interests" TEXT NULL ,
+ PRIMARY KEY ("id") )
+ COMMAND
+ non_query(command)
+
+ command = <<-COMMAND.gsub(/\n\s*/, " ")
+ CREATE TABLE "activity" (
+ "id" CHAR(24) NOT NULL ,
+ "user_id" CHAR(24) NULL ,
+ "name" VARCHAR(255) NULL ,
+ "due_date" DATETIME NULL ,
+ PRIMARY KEY ("id") )
+ COMMAND
+ non_query(command)
+
+ command = <<-COMMAND.gsub(/\n\s*/, " ")
+ CREATE TABLE "organization" (
+ "id" CHAR(24) NOT NULL ,
+ "disabled_date" DATETIME NULL ,
+ PRIMARY KEY ("id") )
+ COMMAND
+ non_query(command)
+end
+
+def export_dbc
+ $db_user ||= 'root'
+ @export_dbc ||= DataObjects::Connection.new("mysql://root@localhost/#{$db_name}")
+end
@@ -0,0 +1,55 @@
+def create_export_db(name)
+ $db_adapter = 'postgres'
+ dbc = DataObjects::Connection.new("postgres://localhost/postgres")
+ dbc.create_command("DROP DATABASE IF EXISTS #{name}").execute_non_query
+ dbc.create_command("CREATE DATABASE #{name}").execute_non_query
+ create_export_tables
+ dbc.dispose
+end
+
+def drop_export_test_db(name)
+ @export_dbc.dispose if @export_dbc
+ dbc = DataObjects::Connection.new("postgres://localhost/postgres")
+ dbc.create_command("DROP DATABASE IF EXISTS #{name}").execute_non_query
+ dbc.dispose
+end
+
+def create_export_tables
+ command = <<-COMMAND.gsub(/\n\s*/, " ")
+ CREATE TABLE "user" (
+ "id" CHAR(24) NOT NULL ,
+ "organization_id" CHAR(24) NOT NULL ,
+ "name" VARCHAR(255) NULL ,
+ "gender" CHAR(1) NULL ,
+ "dob" TIMESTAMP NULL ,
+ "awesome" BOOLEAN NULL ,
+ "fat" BOOLEAN NULL ,
+ "symbolic" VARCHAR(255) NULL ,
+ "interests" TEXT NULL ,
+ PRIMARY KEY ("id") )
+ COMMAND
+ non_query(command)
+
+ command = <<-COMMAND.gsub(/\n\s*/, " ")
+ CREATE TABLE "activity" (
+ "id" CHAR(24) NOT NULL ,
+ "user_id" CHAR(24) NULL ,
+ "name" VARCHAR(255) NULL ,
+ "due_date" TIMESTAMP NULL ,
+ PRIMARY KEY ("id") )
+ COMMAND
+ non_query(command)
+
+ command = <<-COMMAND.gsub(/\n\s*/, " ")
+ CREATE TABLE "organization" (
+ "id" CHAR(24) NOT NULL ,
+ "disabled_date" TIMESTAMP NULL ,
+ PRIMARY KEY ("id") )
+ COMMAND
+ non_query(command)
+end
+
+def export_dbc
+ $db_user ||= ''
+ @export_dbc ||= DataObjects::Connection.new("postgres://localhost/#{$db_name}")
+end
Oops, something went wrong.

0 comments on commit 20e6158

Please sign in to comment.