merged threading support to master and fixed ruby 2.4 integer data ty…

…pe issues bumped up version to 0.8.1
sunitparekh · Aug 19, 2017 · ef2b2d2 · ef2b2d2
1 parent 0eaeb3c
commit ef2b2d2
Show file tree

Hide file tree

Showing 25 changed files with 251 additions and 178 deletions.
diff --git a/README.md b/README.md
@@ -70,6 +70,10 @@ Postgresql database having **composite primary key**
 
 ## Changelog
 
+#### 0.8.1 (Aug 19, 2017)
+1. Multi-threading support added by [stanislav-tyutin](https://github.com/stanislav-tyutin) using Pull Request.  
+2. Fixed to work with Ruby 2.4.x, issue with Integer data type
+
 #### 0.8.0 (Oct 31, 2016)
 1. Upgraded to rails 5.x 
 

diff --git a/data-anonymization.gemspec b/data-anonymization.gemspec
@@ -25,6 +25,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency('rgeo', '~> 0.5')
   gem.add_dependency('rgeo-geojson', '~> 0.4')
   gem.add_dependency('powerbar', '~> 1.0')
-  gem.add_dependency('parallel', '~> 1.9')
-  gem.add_dependency('thor', '~> 0.19')
+  gem.add_dependency('parallel', '~> 1.12')
+  gem.add_dependency('thor', '~> 0.20')
 end
diff --git a/examples/blacklist_dsl.rb b/examples/blacklist_dsl.rb
@@ -1,4 +1,4 @@
-system "bundle exec ruby examples/whitelist_dsl.rb"
+system 'bundle exec ruby examples/whitelist_dsl.rb'
 
 require 'data-anonymization'
 
@@ -18,7 +18,7 @@
     anonymize('City').using FieldStrategy::RandomCity.region_US
     anonymize('State').using FieldStrategy::RandomProvince.region_US
     anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
-    anonymize('Country') {|field| "USA" }
+    anonymize('Country') {|field| 'USA'}
     anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
@@ -33,7 +33,7 @@
     anonymize('City').using FieldStrategy::RandomCity.region_US
     anonymize('State').using FieldStrategy::RandomProvince.region_US
     anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
-    anonymize('Country') {|field| "USA" }
+    anonymize('Country') {|field| 'USA'}
     anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
   end

diff --git a/examples/mongodb_blacklist_dsl.rb b/examples/mongodb_blacklist_dsl.rb
@@ -4,20 +4,20 @@
 DataAnon::Utils::Logging.logger.level = Logger::INFO
 Mongo::Logger.logger.level = Logger::WARN
 
-Mongo::Client.new("mongodb://localhost/test").database.drop
-system "mongoimport --host=127.0.0.1 -d test --drop -c users --jsonArray ./sample-data/mongo/users.json"
-system "mongoimport --host=127.0.0.1 -d test --drop -c plans --jsonArray ./sample-data/mongo/plans.json"
+Mongo::Client.new('mongodb://localhost/test').database.drop
+system 'mongoimport --host=127.0.0.1 -d test --drop -c users --jsonArray ./sample-data/mongo/users.json'
+system 'mongoimport --host=127.0.0.1 -d test --drop -c plans --jsonArray ./sample-data/mongo/plans.json'
 
 
 database 'test' do
   strategy DataAnon::Strategy::MongoDB::Blacklist
-  source_db :mongodb_uri => "mongodb://localhost/test", :database => 'test'
+  source_db :mongodb_uri => 'mongodb://localhost/test', :database => 'test'
 
   collection 'users' do
     anonymize('date_of_birth').using FieldStrategy::TimeDelta.new(5,30)
     anonymize('user_id').using FieldStrategy::StringTemplate.new('user-#{row_number}')
     anonymize('email').using FieldStrategy::RandomMailinatorEmail.new
-    anonymize('password') { |field| "password" }
+    anonymize('password') { |field| 'password'}
     anonymize('first_name').using FieldStrategy::RandomFirstName.new
     anonymize('last_name').using FieldStrategy::RandomLastName.new
   end

diff --git a/examples/mongodb_whitelist_dsl.rb b/examples/mongodb_whitelist_dsl.rb
@@ -5,22 +5,22 @@
 Mongo::Logger.logger.level = Logger::WARN
 
 
-Mongo::Client.new("mongodb://localhost/test").database.drop
-Mongo::Client.new("mongodb://localhost/dest").database.drop
-system "mongoimport --host=127.0.0.1 -d test --drop -c users --jsonArray ./sample-data/mongo/users.json"
-system "mongoimport --host=127.0.0.1 -d test --drop -c plans --jsonArray ./sample-data/mongo/plans.json"
+Mongo::Client.new('mongodb://localhost/test').database.drop
+Mongo::Client.new('mongodb://localhost/dest').database.drop
+system 'mongoimport --host=127.0.0.1 -d test --drop -c users --jsonArray ./sample-data/mongo/users.json'
+system 'mongoimport --host=127.0.0.1 -d test --drop -c plans --jsonArray ./sample-data/mongo/plans.json'
 
 database 'test' do
   strategy DataAnon::Strategy::MongoDB::Whitelist
-  source_db :mongodb_uri => "mongodb://localhost/test", :database => 'test'
-  destination_db :mongodb_uri => "mongodb://localhost/dest", :database => 'dest'
+  source_db :mongodb_uri => 'mongodb://localhost/test', :database => 'test'
+  destination_db :mongodb_uri => 'mongodb://localhost/dest', :database => 'dest'
 
   collection 'users' do
     whitelist '_id','failed_attempts','updated_at'
     anonymize('date_of_birth').using FieldStrategy::TimeDelta.new(5,30)
     anonymize('user_id').using FieldStrategy::StringTemplate.new('user-#{row_number}')
     anonymize('email').using FieldStrategy::RandomMailinatorEmail.new
-    anonymize('password') { |field| "password" }
+    anonymize('password') { |field| 'password'}
     anonymize('first_name').using FieldStrategy::RandomFirstName.new
     anonymize('last_name').using FieldStrategy::RandomLastName.new
     anonymize 'password_reset_answer','password_reset_question'

diff --git a/examples/whitelist_dsl.rb b/examples/whitelist_dsl.rb
@@ -1,4 +1,4 @@
-system "rake empty_dest" # clean destination database on every call
+system 'rake empty_dest' # clean destination database on every call
 
 require 'data-anonymization'
 
@@ -15,7 +15,7 @@
     primary_key 'GenreId'
     whitelist 'GenreId'
     anonymize 'Name' do |field|
-      field.value + " test"
+      field.value + ' test'
     end
   end
 
@@ -39,7 +39,7 @@
     anonymize('City').using FieldStrategy::RandomCity.region_US
     anonymize('State').using FieldStrategy::RandomProvince.region_US
     anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
-    anonymize('Country') {|field| "USA" }
+    anonymize('Country') {|field| 'USA'}
     anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
   end
@@ -56,7 +56,7 @@
     anonymize('City').using FieldStrategy::RandomCity.region_US
     anonymize('State').using FieldStrategy::RandomProvince.region_US
     anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
-    anonymize('Country') {|field| "USA" }
+    anonymize('Country') {|field| 'USA'}
     anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
     anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')

diff --git a/examples/whitelist_dsl_threads.rb b/examples/whitelist_dsl_threads.rb
@@ -0,0 +1,66 @@
+system 'rake empty_dest' # clean destination database on every call
+
+require 'data-anonymization'
+
+DataAnon::Utils::Logging.logger.level = Logger::INFO
+
+database 'Chinook' do
+  strategy DataAnon::Strategy::Whitelist
+  source_db :adapter => 'sqlite3', :database => 'sample-data/chinook.sqlite'
+  destination_db :adapter => 'sqlite3', :database => 'sample-data/chinook-empty.sqlite'
+
+  default_field_strategies :string => FieldStrategy::StringTemplate.new('Sunit #{row_number} Parekh')
+
+  table 'Genre' do
+    primary_key 'GenreId'
+    whitelist 'GenreId'
+    anonymize 'Name' do |field|
+      field.value + ' test'
+    end
+  end
+
+  table 'MediaType' do
+    primary_key 'MediaTypeId'
+    anonymize('MediaTypeId') { |field| field.value } # same as whitelist
+    anonymize('Name').using FieldStrategy::StringTemplate.new('Media Type #{row_number}')
+
+  end
+
+  table 'Customer' do
+    primary_key 'CustomerId'
+    thread_num 5  # thread_num
+
+    whitelist 'CustomerId', 'SupportRepId', 'Company'
+    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('FirstName').using FieldStrategy::RandomFirstName.new
+    anonymize('LastName').using FieldStrategy::RandomLastName.new
+    anonymize('Address').using FieldStrategy::RandomAddress.region_US
+    anonymize('City').using FieldStrategy::RandomCity.region_US
+    anonymize('State').using FieldStrategy::RandomProvince.region_US
+    anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
+    anonymize('Country') {|field| 'USA'}
+    anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+  end
+
+  table 'Employee' do
+    thread_num 5  # thread_num
+
+    whitelist 'EmployeeId', 'ReportsTo', 'Title'
+    anonymize('BirthDate').using FieldStrategy::DateTimeDelta.new(1, 1)
+    anonymize('FirstName').using FieldStrategy::RandomFirstName.new
+    anonymize('LastName').using FieldStrategy::RandomLastName.new
+    anonymize('HireDate').using FieldStrategy::DateTimeDelta.new(2, 0)
+    anonymize('Address').using FieldStrategy::RandomAddress.region_US
+    anonymize('City').using FieldStrategy::RandomCity.region_US
+    anonymize('State').using FieldStrategy::RandomProvince.region_US
+    anonymize('PostalCode').using FieldStrategy::RandomZipcode.region_US
+    anonymize('Country') {|field| 'USA'}
+    anonymize('Phone').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Fax').using FieldStrategy::RandomPhoneNumber.new
+    anonymize('Email').using FieldStrategy::StringTemplate.new('test+#{row_number}@gmail.com')
+  end
+
+
+
+end
diff --git a/lib/core/field.rb b/lib/core/field.rb
@@ -3,7 +3,7 @@ module Core
 
     class Field
 
-      def initialize name, value, row_number, ar_record, table_name = "unknown"
+      def initialize name, value, row_number, ar_record, table_name = 'unknown'
         @name = name
         @value = value
         @row_number = row_number

diff --git a/lib/core/table_errors.rb b/lib/core/table_errors.rb
@@ -11,7 +11,7 @@ def initialize table_name
 
       def log_error record, exception
         @errors << { :record => record, :exception => exception}
-        raise "Reached limit of error for a table" if @errors.length > 100
+        raise 'Reached limit of error for a table' if @errors.length > 100
       end
 
       def errors

diff --git a/lib/data-anonymization.rb b/lib/data-anonymization.rb
@@ -1,28 +1,28 @@
-require "version"
+require 'version'
 
-require "utils/logging"
-require "utils/random_int"
-require "utils/random_float"
-require "utils/random_string"
-require "utils/random_string_chars_only"
-require "utils/geojson_parser"
-require "utils/progress_bar"
-require "utils/parallel_progress_bar"
-require "utils/resource"
-require "utils/template_helper"
-require "parallel/table"
-require "core/database"
-require "core/fields_missing_strategy"
-require "thor/helpers/rdbms_dsl_generator"
-require "core/field"
-require "core/table_errors"
-require "strategy/strategies"
-require "utils/database"
-require "core/dsl"
+require 'utils/logging'
+require 'utils/random_int'
+require 'utils/random_float'
+require 'utils/random_string'
+require 'utils/random_string_chars_only'
+require 'utils/geojson_parser'
+require 'utils/progress_bar'
+require 'utils/parallel_progress_bar'
+require 'utils/resource'
+require 'utils/template_helper'
+require 'parallel/table'
+require 'core/database'
+require 'core/fields_missing_strategy'
+require 'thor/helpers/rdbms_dsl_generator'
+require 'core/field'
+require 'core/table_errors'
+require 'strategy/strategies'
+require 'utils/database'
+require 'core/dsl'
 
 begin
   require 'mongo'
-  require "thor/helpers/mongodb_dsl_generator"
+  require 'thor/helpers/mongodb_dsl_generator'
 rescue LoadError
-  "Ignoring the mongodb specific libraries if monog driver is not specified in gem"
+  'Ignoring the mongodb specific libraries if monog driver is not specified in gem'
 end
diff --git a/lib/strategy/field/default_anon.rb b/lib/strategy/field/default_anon.rb
@@ -5,6 +5,7 @@ module Field
       class DefaultAnon
 
         DEFAULT_STRATEGIES = {:string => FieldStrategy::RandomString.new,
+                              :integer => FieldStrategy::RandomIntegerDelta.new(5),
                               :fixnum => FieldStrategy::RandomIntegerDelta.new(5),
                               :bignum => FieldStrategy::RandomIntegerDelta.new(5000),
                               :float => FieldStrategy::RandomFloatDelta.new(5.0),

diff --git a/lib/strategy/strategies.rb b/lib/strategy/strategies.rb
@@ -9,5 +9,5 @@
   require 'strategy/mongodb/whitelist'
   require 'strategy/mongodb/blacklist'
 rescue LoadError
-  "Ignoring the mongodb specific libraries if monog driver is not specified in gem"
+  'Ignoring the mongodb specific libraries if monog driver is not specified in gem'
 end
diff --git a/lib/utils/template_helper.rb b/lib/utils/template_helper.rb
@@ -5,9 +5,11 @@ class TemplateHelper
       def self.source_connection_specs_rdbms config_hash
 
         config_hash.keys.reject{|key| config_hash[key].nil? }.collect { |key|
-          if ((config_hash[key].class.to_s.downcase == "string"))
+          if ((config_hash[key].class.to_s.downcase == 'string'))
             ":#{key} => '#{config_hash[key]}'"
-          elsif ((config_hash[key].class.to_s.downcase == "fixnum"))
+          elsif ((config_hash[key].class.to_s.downcase == 'integer'))
+            ":#{key} => #{config_hash[key]}"
+          elsif ((config_hash[key].class.to_s.downcase == 'fixnum'))
             ":#{key} => #{config_hash[key]}"
           end
         }.join ', '

diff --git a/lib/version.rb b/lib/version.rb
@@ -1,3 +1,3 @@
 module DataAnonymization
-  VERSION = '0.8.0'
+  VERSION = '0.8.1'
 end