Permalink
Browse files

Merge pull request #1 from Tereci/master

Seems ok
  • Loading branch information...
2 parents 8a11d27 + 6ee1884 commit 19aaef6561fea16b12a5ac1ff1c091044d3e35aa @fluke777 committed May 11, 2012
Showing with 122 additions and 46 deletions.
  1. +106 −41 bin/es
  2. +16 −5 lib/es.rb
View
147 bin/es
@@ -140,6 +140,10 @@ command :load_deleted do |c|
c.desc 'Base files directory. If specified it will ignore specific files and it will pick up all files in this directory with pattern load*.json'
c.default_value nil
c.flag [:b, :basedir]
+
+ c.desc 'Compatibility mode. If set to true deleted records will be loaded old style with type isDeleted. Otherwise deleted records will be loaded with type attribute and DeletedAt field will be added.'
+ c.default_value false
+ c.flag [:c, :compatibility]
c.action do |global_options,options,args|
@@ -152,6 +156,9 @@ command :load_deleted do |c|
filenames = Dir::glob("#{base_dir}/gen_load*.json")
end
+ compatibility_mode = options[:compatibility] || false
+ deleted_type = compatibility_mode ? "isDeleted" : "attribute"
+
filenames.each do |load_config_file|
load_config = Es::Helpers.load_config(load_config_file)
load = Es::Load.parse(load_config)
@@ -167,28 +174,30 @@ command :load_deleted do |c|
:fields => [
Es::Field.new('Id', 'recordid'),
Es::Field.new('Timestamp', 'timestamp'),
- Es::Field.new('IsDeleted', 'attribute')
+ Es::Field.new('IsDeleted', deleted_type)
]
})
e.load(PID)
- deleted_with_time = "#{source_dir}/#{deleted_filename}".gsub(/\.csv$/, '_del.csv')
- FasterCSV.open(deleted_with_time, 'w') do |csv|
- csv << ['Id', 'Timestamp', 'DeletedAt']
- FasterCSV.foreach("#{source_dir}/#{deleted_filename}", :headers => true, :return_headers => false) do |row|
- csv << row.values_at('Id', 'Timestamp', 'Timestamp')
+ if !compatibility_mode
+ deleted_with_time = "#{source_dir}/#{deleted_filename}".gsub(/\.csv$/, '_del.csv')
+ FasterCSV.open(deleted_with_time, 'w') do |csv|
+ csv << ['Id', 'Timestamp', 'DeletedAt']
+ FasterCSV.foreach("#{source_dir}/#{deleted_filename}", :headers => true, :return_headers => false) do |row|
+ csv << row.values_at('Id', 'Timestamp', 'Timestamp')
+ end
end
- end
- e1 = Es::Entity.new(entity.name, {
- :file => deleted_with_time,
- :fields => [
- Es::Field.new('Id', 'recordid'),
- Es::Field.new('Timestamp', 'timestamp'),
- Es::Field.new('DeletedAt', 'time')
- ]
- })
- e1.load(PID)
+ e1 = Es::Entity.new(entity.name, {
+ :file => deleted_with_time,
+ :fields => [
+ Es::Field.new('Id', 'recordid'),
+ Es::Field.new('Timestamp', 'timestamp'),
+ Es::Field.new('DeletedAt', 'time')
+ ]
+ })
+ e1.load(PID)
+ end
end
end
end
@@ -293,54 +302,110 @@ command :extract do |c|
end
end
+desc 'Generate extract config template'
+command :generate_extract do |c|
+ c.desc 'Base files directory. If specified it will ignore specific files and it will pick up all files in this directory with pattern load*.json'
+ c.default_value nil
+ c.flag [:b, :basedir]
+
+ c.action do |global_options,options,args|
+ base_dir = options[:basedir]
+ fail "You need to specify base dir" if base_dir.nil?
+
+ base_filenames = Dir::glob("#{base_dir}/gen_load_*.json")
+ # build one giant load config
+ base_entities = base_filenames.reduce([]) do |memo, filename|
+ fail "File #{filename} cannot be found" unless File.exist?(filename)
+ load_config = Es::Helpers.load_config(filename)
+ load = Es::Load.parse(load_config)
+ memo.concat(load.entities)
+ end
+ hyper_load = Es::Load.new(base_entities)
+ entity_names = hyper_load.entities.map {|e| e.name}.uniq
+
+ entity_names.each do |entity_name|
+ entity = hyper_load.get_merged_entity_for(entity_name)
+
+ File.open(base_dir+"/gen_extract_"+entity.name+".json", 'w') do |f|
+ f.write JSON.pretty_generate(entity.to_extract_config)
+ end
+ end
+
+ end
+end
desc 'Generate base conifg template'
command :generate_base do |c|
- c.desc 'Name of the entity.'
+ c.desc 'Name of the entity. If inputdir is also specified all generated base files will have this entity set. If you want to set entity for each file according to the file name do not provide this parameter.'
c.default_value nil
c.flag [:e, :entity]
c.desc 'Input file.'
c.default_value nil
c.flag [:i, :input]
-
+
c.desc 'Output filename. If not provided it print to STDOUT.'
c.default_value nil
c.flag [:o, :output]
+
+ c.desc 'Input files directory. If specified it will ignore specific file and it will pick up all files in this directory with pattern *.csv'
+ c.default_value nil
+ c.flag [:s, :inputdir]
+
+ c.desc 'Base files directory. If this and inputdir are specified base JSONs will be generated into the directory.'
+ c.default_value nil
+ c.flag [:b, :basedir]
c.action do |global_options,options,args|
- entity_name = options[:entity]
+ entity = options[:entity]
input_filename = options[:input]
+ input_dir = options[:inputdir]
output_filename = options[:output]
+ base_dir = options[:basedir]
- fail "You need to specify input entity name" if entity_name.nil?
- fail "You need to specify input file name" if input_filename.nil?
+ fail "You need to specify input file name or input dir" if input_filename.nil? && input_dir.nil?
- headers = nil
- FasterCSV.foreach(input_filename, :headers => true, :return_headers => true) do |row|
- if row.header_row?
- headers = row.fields
- break
- end
+ if base_dir.nil?
+ input_filenames = [input_filename]
+ else
+ input_filenames = Dir::glob("#{input_dir}/*.csv")
end
-
- load = Es::Load.new([
- Es::Entity.new(entity_name, {
- :file => input_filename,
- :fields => headers.map do |field_name|
- Es::Field.new(field_name, "none")
+
+ input_filenames.each do |input_filename|
+
+ headers = nil
+ FasterCSV.foreach(input_filename, :headers => true, :return_headers => true) do |row|
+ if row.header_row?
+ headers = row.fields
+ break
end
- })
- ])
+ end
- config = JSON.pretty_generate(load.to_config)
- if output_filename.nil?
- puts config
- else
- File.open(output_filename, 'w') do |f|
- f.write config
+ entity_name = entity || File.basename(input_filename, ".csv")
+ load = Es::Load.new([
+ Es::Entity.new(entity_name, {
+ :file => input_filename,
+ :fields => headers.map do |field_name|
+ Es::Field.new(field_name, "none")
+ end
+ })
+ ])
+
+ config = JSON.pretty_generate(load.to_config)
+
+ if input_dir && base_dir
+ File.open(base_dir+"/gen_load_"+entity_name+".json", 'w') do |f|
+ f.write config
+ end
+ elsif input_filename && output_filename
+ File.open(output_filename, 'w') do |f|
+ f.write config
+ end
+ else
+ puts config
end
+
end
end
end
View
21 lib/es.rb
@@ -177,7 +177,7 @@ def validate
end
def to_config
- entities.map {|e| e.to_config}
+ entities.map {|e| e.to_load_config}
end
def to_config_file(filename)
@@ -261,11 +261,22 @@ def to_load_fragment(pid)
}
end
- def to_config
+ def to_load_config
{
:entity => name,
:file => file,
- :fields => fields.map {|f| f.to_config}
+ :fields => fields.map {|f| f.to_load_config}
+ }
+ end
+
+ def to_extract_config
+ {
+ :timezone => timezone,
+ :entities => [{
+ :entity => name,
+ :file => file,
+ :fields => fields.map {|f| f.name}
+ }]
}
end
@@ -323,7 +334,7 @@ class Field
HISTORIC_TYPE = "historicid"
DURATION_TYPE = "duration"
VELOCITY_TYPE = "velocity"
- IS_DELETED_TYPE = 'isDeleted'
+ IS_DELETED_TYPE = "isDeleted"
FIELD_TYPES = [ATTRIBUTE_TYPE, RECORDID_TYPE, DATE_TYPE, TIME_TYPE, FACT_TYPE, TIMESTAMP_TYPE, AUTOINCREMENT_TYPE, SNAPSHOT_TYPE, HID_TYPE, HISTORIC_TYPE, DURATION_TYPE, VELOCITY_TYPE, IS_DELETED_TYPE]
@@ -388,7 +399,7 @@ def to_load_fragment(pid)
}
end
- def to_config
+ def to_load_config
{
:name => name,
:type => (type == 'none' ? '' : type)

0 comments on commit 19aaef6

Please sign in to comment.