Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Add post-ETL hooks. Closes #1725.

No hooks implemented yet, so the API may yet change a bit.
  • Loading branch information...
commit 6c511b9c15bf8cfeb834a5c986a5f3d6ff850a6f 1 parent ac8a1db
@rsutphin rsutphin authored
View
7 CHANGELOG.md
@@ -1,13 +1,18 @@
NCS Navigator MDES Warehouse History
====================================
-0.4.2
+0.5.0
-----
+- Add "post-ETL hooks" to ETL process: objects with callbacks which
+ are executed when the ETL completes. (#1725)
+
- Include the input filename in the name of transformers based on
`VdrXml::Reader`. (#1927)
+
- Exclude parent bundler environment when executing subprocess in
SubprocessTransformer. (#2012)
+
- Strip leading and trailing whitespace from values in one-to-one
transformer. (#2028)
View
1  lib/ncs_navigator/warehouse.rb
@@ -12,6 +12,7 @@ module Warehouse
autoload :DataMapper, 'ncs_navigator/warehouse/data_mapper'
autoload :DatabaseInitializer, 'ncs_navigator/warehouse/database_initializer'
autoload :Models, 'ncs_navigator/warehouse/models'
+ autoload :PostEtlHooks, 'ncs_navigator/warehouse/post_etl_hooks'
autoload :PostgreSQL, 'ncs_navigator/warehouse/postgresql'
autoload :TableModeler, 'ncs_navigator/warehouse/table_modeler'
autoload :Transformers, 'ncs_navigator/warehouse/transformers'
View
30 lib/ncs_navigator/warehouse/configuration.rb
@@ -83,6 +83,36 @@ def add_transformer(candidate)
end
####
+ #### Hooks
+ ####
+
+ ##
+ # @return [Array<#etl_succeeded,#etl_failed>] the configured
+ # post-ETL hooks.
+ def post_etl_hooks
+ @post_etl_hooks ||= []
+ end
+
+ ##
+ # Adds a post-ETL hook to the list for this warehouse instance.
+ #
+ # @return [void]
+ # @param [#etl_succeeded,#etl_failed] the hook
+ def add_post_etl_hook(candidate)
+ expected_methods = [:etl_succeeded, :etl_failed]
+ implemented_methods = expected_methods.select { |m| candidate.respond_to?(m) }
+ if implemented_methods.empty?
+ msg = "#{candidate.inspect} does not have an #{expected_methods.join(' or ')} method."
+ if candidate.respond_to?(:new)
+ msg += " Perhaps you meant #{candidate}.new?"
+ end
+ raise Error, msg
+ else
+ post_etl_hooks << candidate
+ end
+ end
+
+ ####
#### MDES version
####
View
16 lib/ncs_navigator/warehouse/post_etl_hooks.rb
@@ -0,0 +1,16 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse
+ ##
+ # The namespace for post-ETL hook implementations which are provided
+ # with the warehouse.
+ #
+ # A post-ETL hook is an object which responds to either
+ # `etl_succeeded` or `etl_failed` (or both). Each method takes one
+ # argument: the list of {TransformStatus}es describing the ETL
+ # process that was just completed.
+ #
+ # @see Configuration#add_post_etl_hook
+ module PostEtlHooks
+ end
+end
View
13 lib/ncs_navigator/warehouse/transform_load.rb
@@ -53,8 +53,10 @@ def repo.identity_map(model); {}; end
end
if statuses.detect { |s| !s.transform_errors.empty? }
+ dispatch_post_etl_hooks(:etl_failed)
false
else
+ dispatch_post_etl_hooks(:etl_succeeded)
true
end
end
@@ -67,5 +69,16 @@ def build_status_for(transformer, position)
)
end
private :build_status_for
+
+ def dispatch_post_etl_hooks(method)
+ configuration.post_etl_hooks.each do |hook|
+ begin
+ hook.send(method, statuses) if hook.respond_to?(method)
+ rescue => e
+ log.error("Error invoking #{method.inspect} on #{hook.inspect}: #{e.class} #{e}.")
+ end
+ end
+ end
+ private :dispatch_post_etl_hooks
end
end
View
2  lib/ncs_navigator/warehouse/version.rb
@@ -1,5 +1,5 @@
module NcsNavigator
module Warehouse
- VERSION = '0.4.2.pre'
+ VERSION = '0.5.0.pre'
end
end
View
48 spec/ncs_navigator/warehouse/configuration_spec.rb
@@ -41,6 +41,54 @@ def transform
end
end
+ describe 'add_post_etl_hook' do
+ let(:success_hook) {
+ Object.new.tap do |o|
+ class << o
+ def etl_succeeded; end
+ end
+ end
+ }
+
+ let(:failure_hook) {
+ Object.new.tap do |o|
+ class << o
+ def etl_failed; end
+ end
+ end
+ }
+
+ it 'adds a hook with an `etl_succeeded` method' do
+ config.add_post_etl_hook(success_hook)
+ config.post_etl_hooks.should == [success_hook]
+ end
+
+ it 'adds a hook with an `etl_failed` method' do
+ config.add_post_etl_hook(failure_hook)
+ config.post_etl_hooks.should == [failure_hook]
+ end
+
+ describe 'with an object without either etl callback method' do
+ it 'gives a helpful message if the object is constructable' do
+ lambda { config.add_post_etl_hook(String) }.
+ should raise_error('String does not have an etl_succeeded or etl_failed method. Perhaps you meant String.new?')
+ end
+
+ it 'gives a helpful message if the object is an instance' do
+ lambda { config.add_post_etl_hook('not a good hook') }.
+ should raise_error('"not a good hook" does not have an etl_succeeded or etl_failed method.')
+ end
+
+ it 'does not add the object as a hook' do
+ begin
+ config.add_post_etl_hook('not a good hook')
+ rescue
+ end
+ config.post_etl_hooks.should be_empty
+ end
+ end
+ end
+
describe '#mdes_version=' do
context 'for a known version', :slow, :use_mdes, :modifies_warehouse_state do
it 'makes the models available' do
View
109 spec/ncs_navigator/warehouse/transform_load_spec.rb
@@ -162,10 +162,82 @@ def transformer.name; 'provided'; end
end
end
- describe 'sending e-mail' do
- it 'sends on success'
+ describe 'with post-ETL hooks' do
+ let(:hook_a) { RecordingHook.new }
+ let(:hook_success) { RecordingHook.new(:succeeded) }
+ let(:hook_failure) { RecordingHook.new(:failed) }
+ let(:hook_error) {
+ Class.new do
+ def etl_succeeded(ts)
+ fail 'Hook broke'
+ end
+ alias :etl_failed :etl_succeeded
+ end.new
+ }
+
+ let(:hooks_invoked) {
+ [hook_a, hook_success, hook_failure].map(&:invoked?)
+ }
+
+ before do
+ config.add_post_etl_hook(hook_a)
+ config.add_post_etl_hook(hook_success)
+ config.add_post_etl_hook(hook_failure)
+ end
+
+ shared_examples 'all hooks' do
+ before do
+ loader.run
+ end
+
+ it 'sends each hook the statuses' do
+ hook_a.transform_statuses.size.should == 1
+ end
+
+ it 'runs all the hooks even when one fails' do
+ config.post_etl_hooks.unshift hook_error
- it 'sends on failure'
+ loader.run
+
+ hooks_invoked.should == expected_invoke_pattern
+ end
+ end
+
+ describe 'when the transform fails' do
+ let(:expected_invoke_pattern) { [true, false, true] }
+
+ before do
+ config.add_transformer(BlockTransformer.new { |s| fail 'Nope.' })
+ end
+
+ include_examples 'all hooks'
+
+ it 'runs all the hooks that have an etl_failed method' do
+ hooks_invoked.should == expected_invoke_pattern
+ end
+
+ it 'sends each hook the overall status' do
+ hook_a.should be_failure
+ end
+ end
+
+ describe 'when the transform succeeds' do
+ let(:expected_invoke_pattern) { [true, true, false] }
+
+ before do
+ config.add_transformer(BlockTransformer.new { })
+ end
+
+ include_examples 'all hooks'
+
+ it 'runs all the hooks that have an etl_succeeded method' do
+ hooks_invoked.should == expected_invoke_pattern
+ end
+
+ it 'sends each hook the overall status' do
+ hook_a.should be_success
+ end
+ end
end
# This is a crappy test; it would be better if it could be done
@@ -195,5 +267,36 @@ def transform(status)
@block.call(status)
end
end
+
+ class ::RecordingHook
+ attr_reader :transform_statuses
+
+ def initialize(*modes)
+ @invoked = false
+ @modes = modes.empty? ? [:succeeded, :failed] : modes
+
+ @modes.each do |mode|
+ instance_eval <<-RUBY
+ def etl_#{mode}(transform_statuses)
+ @invoked = true
+ @transform_statuses = transform_statuses
+ @success = #{mode == :succeeded}
+ end
+ RUBY
+ end
+ end
+
+ def invoked?
+ @invoked
+ end
+
+ def success?
+ @success
+ end
+
+ def failure?
+ !@success.nil? && !@success
+ end
+ end
end
end
Please sign in to comment.
Something went wrong with that request. Please try again.