Skip to content
Permalink
Browse files

Merge pull request #140 from killbill/kpm-bugfix

KPM import fixes
  • Loading branch information...
pierre committed Apr 3, 2019
2 parents 7b41bde + 0e85ba3 commit f6f46a209ee914ad4e4538003461ddac6fbf287c
Showing with 36 additions and 12 deletions.
  1. +4 −2 kpm/lib/kpm/account.rb
  2. +10 −1 kpm/lib/kpm/database.rb
  3. +1 −4 kpm/lib/kpm/tasks.rb
  4. +21 −5 kpm/spec/kpm/unit_mysql/account_spec.rb
@@ -39,7 +39,7 @@ class Account

DATE_COLUMNS_TO_FIX = ['created_date','updated_date','processing_available_date','effective_date',
'boot_date','start_timestamp','last_access_time','payment_date','original_created_date',
'last_sys_update_date','charged_through_date','bundle_start_date','start_date']
'last_sys_update_date','charged_through_date','bundle_start_date','start_date', 'reference_time']

# round trip constants duplicate record
ROUND_TRIP_EXPORT_IMPORT_MAP = {:accounts => {:id => :accounts_id, :external_key => :accounts_id}, :all => {:account_id => :accounts_id},
@@ -109,6 +109,7 @@ def export_data(account_id = nil)
end

def import_data(source_file, tenant_record_id, skip_payment_methods, round_trip_export_import = false, generate_record_id = false)
source_file = File.expand_path(source_file)

@generate_record_id = generate_record_id
@tenant_record_id = tenant_record_id
@@ -119,7 +120,7 @@ def import_data(source_file, tenant_record_id, skip_payment_methods, round_trip_
end

unless File.exist?(source_file)
raise Interrupt, 'Need to specify a valid file'
raise Interrupt, "File #{source_file} does not exist"
end

@delimiter = sniff_delimiter(source_file) || @delimiter
@@ -275,6 +276,7 @@ def process_import_data(line, table_name, cols_names, skip_payment_methods, rows

row = []

@logger.debug "Processing table_name=#{table_name}, line=#{line}"
cols_names.each_with_index do |col_name, index|
sanitized_value = sanitize(table_name,col_name,cols[index], skip_payment_methods)

@@ -102,7 +102,16 @@ def generate_insert_statement(tables)

rows = []
table[:rows].each do |row|
rows << row.map{|value| value.is_a?(Symbol) ? value.to_s : "'#{value.to_s.gsub(/['"]/, "'" => "\\'", '"' => '\\"')}'" }.join(",")
rows << row.map do |value|
if value.is_a?(Symbol)
value.to_s
else
escaped_value = value.to_s.gsub(/['"]/, "'" => "\\'", '"' => '\\"')
.gsub('\N{LINE FEED}', "\n")
.gsub('\N{VERTICAL LINE}', "|")
"'#{escaped_value}'"
end
end.join(",")
end

value_data = rows.map{|row| "(#{row})" }.join(",")
@@ -496,7 +496,6 @@ def system
:desc => 'Database port'
desc 'account', 'export/import accounts'
def account
logger.info 'Please wait processing the request!!!'
begin
config_file = nil
if options[:killbill_url] && /https?:\/\/[\S]+/.match(options[:killbill_url]).nil?
@@ -570,7 +569,6 @@ def account
:desc => 'Killbill URL ex. http://127.0.0.1:8080'
desc 'tenant_config', 'export all tenant-level configs.'
def tenant_config
logger.info 'Please wait processing the request!!!'
begin

if options[:killbill_url] && /https?:\/\/[\S]+/.match(options[:killbill_url]).nil?
@@ -657,7 +655,6 @@ def tenant_config
:desc => 'A different folder other than the default bundles directory.'
desc 'diagnostic', 'exports and \'zips\' the account data, system, logs and tenant configurations'
def diagnostic
logger.info 'Please wait processing the request!!!'
begin
if options[:account_export] && options[:account_export] == 'account_export'
raise Interrupt,'--account_export, please provide a valid account id'
@@ -711,7 +708,7 @@ def diagnostic

def logger
logger = ::Logger.new(STDOUT)
logger.level = Logger::INFO
logger.level = ENV['KPM_DEBUG'] ? Logger::DEBUG : Logger::INFO
logger
end
end
@@ -12,9 +12,9 @@
let(:account_id_invalid) {SecureRandom.uuid}
let(:dummy_data) {
"-- accounts record_id|id|external_key|email|name|first_name_length|currency|billing_cycle_day_local|parent_account_id|is_payment_delegated_to_parent|payment_method_id|time_zone|locale|address1|address2|company_name|city|state_or_province|country|postal_code|phone|notes|migrated|is_notified_for_invoices|created_date|created_by|updated_date|updated_by|tenant_record_id\n"\
"5|#{dummy_account_id}|#{dummy_account_id}|willharnet@example.com|Will Harnet||USD|0||||UTC||||||||||||false|2017-04-03T15:50:14.000+0000|demo|2017-04-05T15:01:39.000+0000|Killbill::Stripe::PaymentPlugin|2\n"\
"5|#{dummy_account_id}|#{dummy_account_id}|willharnet@example.com|Will Harnet||USD|0||||UTC||||Company\\N{VERTICAL LINE}\\N{LINE FEED}Name||||||||false|2017-04-03T15:50:14.000+0000|demo|2017-04-05T15:01:39.000+0000|Killbill::Stripe::PaymentPlugin|2\n"\
"-- account_history record_id|id|target_record_id|external_key|email|name|first_name_length|currency|billing_cycle_day_local|parent_account_id|payment_method_id|is_payment_delegated_to_parent|time_zone|locale|address1|address2|company_name|city|state_or_province|country|postal_code|phone|notes|migrated|is_notified_for_invoices|change_type|created_by|created_date|updated_by|updated_date|tenant_record_id\n"\
"3|#{SecureRandom.uuid}|5|#{dummy_account_id}|willharnet@example.com|Will Harnet||USD|0||||UTC||||||||||||false|INSERT|demo|2017-04-03T15:50:14.000+0000|demo|2017-04-03T15:50:14.000+0000|2\n"
"3|#{SecureRandom.uuid}|5|#{dummy_account_id}|willharnet@example.com|Will Harnet||USD|0||||UTC||||Company\\N{VERTICAL LINE}\\N{LINE FEED}Name||||||||false|INSERT|demo|2017-04-03T15:50:14.000+0000|demo|2017-04-03T15:50:14.000+0000|2\n"
}
let(:cols_names) {dummy_data.split("\n")[0].split(" ")[2]}
let(:cols_data) {dummy_data.split("\n")[1]}
@@ -303,7 +303,7 @@
end

it 'when importing data with no file' do
expect{account_class.import_data(dummy_data_file,nil,true,false,true) }.to raise_error(Interrupt,'Need to specify a valid file')
expect{account_class.import_data(dummy_data_file,nil,true,false,true) }.to raise_error(Interrupt, "File #{dummy_data_file} does not exist")
end

it 'when importing data with new record_id' do
@@ -312,6 +312,8 @@
end
expect{account_class.import_data(dummy_data_file,nil,true,false,true) }.not_to raise_error(Interrupt)

verify_data(dummy_account_id)

row_count_inserted = delete_statement('accounts','id',dummy_account_id)
expect(row_count_inserted).to eq('1')
row_count_inserted = delete_statement('account_history','external_key',dummy_account_id)
@@ -324,6 +326,8 @@
end
expect{account_class.import_data(dummy_data_file,nil,true,false,false) }.not_to raise_error(Interrupt)

verify_data(dummy_account_id)

row_count_inserted = delete_statement('accounts','id',dummy_account_id)
expect(row_count_inserted).to eq('1')
row_count_inserted = delete_statement('account_history','external_key',dummy_account_id)
@@ -336,6 +340,8 @@
end
expect{account_class.import_data(dummy_data_file,10,true,false,true) }.not_to raise_error(Interrupt)

verify_data(dummy_account_id)

row_count_inserted = delete_statement('accounts','id',dummy_account_id)
expect(row_count_inserted).to eq('1')
row_count_inserted = delete_statement('account_history','external_key',dummy_account_id)
@@ -349,6 +355,8 @@
expect{account_class.import_data(dummy_data_file,10,true,true,true) }.not_to raise_error(Interrupt)
new_account_id = account_class.instance_variable_get(:@tables_id)

verify_data(new_account_id['accounts_id'])

row_count_inserted = delete_statement('accounts','id',new_account_id['accounts_id'])
expect(row_count_inserted).to eq('1')
row_count_inserted = delete_statement('account_history','external_key',new_account_id['accounts_id'])
@@ -389,8 +397,16 @@ def creating_account_with_client

$account_id
end

def delete_statement(table_name,column_name,account_id)

def verify_data(account_id)
response = `#{mysql_cli} #{db_name} -e "select company_name FROM accounts WHERE id = '#{account_id}';" 2>&1`
response_msg = response.split("\n")
company_name = response_msg[response_msg.size - 1]

expect(company_name).to eq("Company|\\nName")
end

def delete_statement(table_name,column_name,account_id)
response = `#{mysql_cli} #{db_name} -e "DELETE FROM #{table_name} WHERE #{column_name} = '#{account_id}'; SELECT ROW_COUNT();" 2>&1`
response_msg = response.split("\n")
row_count_inserted = response_msg[response_msg.size - 1]

0 comments on commit f6f46a2

Please sign in to comment.
You can’t perform that action at this time.