Permalink
Browse files

Etl delays execution of code until #run(:file) is called

  • Loading branch information...
1 parent 97eb68b commit d93bdbe7894ecbeb97c0642ae80cc0affdac4890 Mateusz Sojka committed Mar 25, 2012
View
@@ -4,17 +4,17 @@ class Etl
attr_accessor :data, :import_attributes, :field_definitions
attr_accessor :extractor, :loader, :transformer, :fields
- def initialize(data, setup = nil)
- @data = data
+ def initialize
@import_attributes = {}
@field_definitions = {}
-
- eval setup unless setup.nil?
end
- def run
- while row = @extractor.next do
- @data = row
+ def run(file)
+ etl_setup = file.is_a?(String) ? file : File.open(file).read
+
+ instance_eval etl_setup
+
+ while(@data = @extractor.next) do
@transformer.instance_eval(&@fields)
@loader.write(@import_attributes)
end
@@ -27,7 +27,7 @@ def source(adapter, &block)
end
def fields(&block)
- @transformer = FieldTransformer.new(self)
+ @transformer ||= FieldTransformer.new(self)
@fields = block
end
@@ -48,7 +48,7 @@
end
it 'writes the content to a file' do
- @adapter.write(['first', 'last', 'age'])
+ @adapter.write(0 => 'first', 1 => 'last', 2 => 'age')
@written_content = File.open(@adapter.loader.destination_file_path, 'r').read
@written_content.should == "first,last,age\n"
end
@@ -1,145 +1,129 @@
require 'spec_helper'
describe Transformerb::Etl do
- describe '.new(:data, :setup)' do
+ describe '#fields' do
before do
- @data = {
- 'name' => 'Lionel',
- 'Last name' => 'Messi',
- 'Fifas best' => '3',
- 'top_score' => 5,
- 'gender' => 'Man',
- 'job' => 'football player',
- 'locality' => 'Longer than allowed',
- 'region' => nil
- }
+ class DummyExtractor; end
+ class DummyLoader; end
+
+ @etl = Transformerb::Etl.new
+
+ @etl.extractor = DummyExtractor.new
+ data = { 'name' => 'Lionel', 'Last name' => 'Messi' }
+ @etl.extractor.stub!(:next).and_return(data, nil)
+
+ @etl.loader = DummyLoader.new
+ @etl.loader.stub!(:write).and_return nil
@setup =<<-EOF
fields do
take 'name'
take 'Last name'
end
EOF
-
- @etl = Transformerb::Etl.new(@data, @setup)
end
- describe '#fields' do
- before do
- class DummyExtractor
-
- def initialize
- @value = { 'name' => 'Lionel', 'Last name' => 'Messi' }
- end
+ it 'sets the import_attributes correctly' do
+ @etl.run(@setup)
- def next
- return_value = @value
- @value = nil
+ @etl.import_attributes[:name].should == 'Lionel'
+ @etl.import_attributes[:last_name].should == 'Messi'
+ end
+ end
- return_value
+ describe '#source' do
+ before do
+ @setup =<<-EOF
+ source CSV do
+ file 'spec/lib/transformerb/adapters/spec.csv'
+ parser_config do |config|
+ config.headers = :first_row
end
end
- @etl.extractor = DummyExtractor.new
- @etl.loader.stub!(:write).and_return nil
- end
+ fields do
+ take 'name'
+ take 'Last name'
+ end
+ EOF
- it 'sets the import_attributes correctly' do
- @etl.run
+ @etl = Transformerb::Etl.new
- @etl.import_attributes[:name].should == 'Lionel'
- @etl.import_attributes[:last_name].should == 'Messi'
- end
+ @etl.loader = DummyLoader.new
+ @etl.loader.stub!(:write).and_return nil
end
- describe '#source' do
- before do
- setup =<<-EOF
- source CSV do
- file 'spec/lib/transformerb/adapters/spec.csv'
- parser_config do |config|
- config.headers = :first_row
- end
- end
-
- fields do
- take 'name'
- take 'Last name'
- end
- EOF
-
- @etl = Transformerb::Etl.new(@data, setup)
- end
-
- it 'uses the source (extract) correctly' do
- @etl.extractor.should be_a(Transformerb::Adapters::Csv::Extractor)
- end
+ it 'uses the source (extract) correctly' do
+ @etl.run(@setup)
+ @etl.extractor.should be_a(Transformerb::Adapters::Csv::Extractor)
end
- describe '#load' do
- before do
- setup =<<-EOF
- source CSV do
- file 'spec/lib/transformerb/adapters/spec.csv'
- parser_config do |config|
- config.headers = :first_row
- end
- end
+ end
- fields do
- take 'name'
- take 'Last name'
- end
+ describe '#load' do
+ before do
+ @setup =<<-EOF
+ fields do
+ take 'name'
+ take 'Last name'
+ end
- load CSV do
- file 'spec/lib/transformerb/adapters/output_spec.csv'
- end
- EOF
+ load CSV do
+ file '/tmp/output_spec.csv'
+ end
+ EOF
+
+ @etl = Transformerb::Etl.new
- @etl = Transformerb::Etl.new(@data, setup)
- end
+ @etl.extractor = DummyExtractor.new
+ data = { 'name' => 'Lionel', 'Last name' => 'Messi' }
+ @etl.extractor.stub!(:next).and_return(data, nil)
+ end
- it 'sets up the correct output (load)' do
- @etl.loader.should be_a(Transformerb::Adapters::Csv::Loader)
- end
+ it 'sets up the correct output (load)' do
+ @etl.run(@setup)
+ @etl.loader.should be_a(Transformerb::Adapters::Csv::Loader)
end
- describe '#run' do
- before do
- setup =<<-EOF
- source CSV do
- file 'spec/lib/transformerb/adapters/spec.csv'
- parser_config do |config|
- config.headers = :first_row
- end
- end
+ end
- fields do
- take 'first name'
- take 'last'
+ describe '#run' do
+ before do
+ @setup =<<-EOF
+ source CSV do
+ file 'spec/lib/transformerb/adapters/spec.csv'
+ parser_config do |config|
+ config.headers = :first_row
end
+ end
- load CSV do
- file 'spec/lib/transformerb/adapters/output_spec.csv'
- end
- EOF
+ fields do
+ take 'first name'
+ take 'last'
+ end
+
+ load CSV do
+ file 'spec/lib/transformerb/adapters/output_spec.csv'
+ end
+ EOF
- @etl = Transformerb::Etl.new(@data, setup)
- end
+ @file_path = 'spec/lib/transformerb/adapters/output_spec.csv'
- it 'writes the result of the transformation' do
- @etl.run
+ @etl = Transformerb::Etl.new
+ end
- csv_file_content = File.open('spec/lib/transformerb/adapters/output_spec.csv').read
- csv_file_content.should == "Andres,Iniesta\nVictor,Valdes\n"
- end
+ it 'writes the result of the transformation' do
+ @etl.run(@setup)
- after do
- system('rm spec/lib/transformerb/adapters/output_spec.csv')
- end
+ csv_file_content = File.open(@file_path).read
+ csv_file_content.should == "Andres,Iniesta\nVictor,Valdes\n"
end
+ after do
+ FileUtils.remove_file(@file_path) if File.exists?(@file_path)
+ end
end
+
end
Oops, something went wrong.

0 comments on commit d93bdbe

Please sign in to comment.