Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Adding a Job object for collecting a definition and turning it into a…

… full munging process.
  • Loading branch information...
commit 232d702ceabb5abf3a40c4cf943e10caa1a046c4 1 parent 5126729
@JEG2 authored
View
94 lib/mungr/job.rb
@@ -0,0 +1,94 @@
+# encoding: UTF-8
+
+require "mungr/reader"
+require "mungr/munge"
+require "mungr/writer"
+
+module Mungr
+ #
+ # A Job object is used to gather Reader, Munge, and Writer objects as a
+ # munging process is defined. After all elements of the process have been
+ # gathered, this Job is used to build() the correct object tree which can be
+ # used to run() the process.
+ #
+ class Job
+ #
+ # Creates an empty Job. You then add Readers, Munges, and Writers needed
+ # and build() the final process. This generally looks something like the
+ # following:
+ #
+ # job = Job.new
+ # reader = Reader.new do |r|
+ # # ...
+ # end
+ # munge = Munge.new do |m|
+ # # ...
+ # end
+ # writer = Writer.new do |w|
+ # # ...
+ # end
+ # job << reader << munge << writer
+ # job.build.run
+ #
+ def initialize
+ @readers = Array.new
+ @munges = Array.new
+ @writers = Array.new
+ end
+
+ #
+ # Adds a Reader, Munge, or Writer that will become a part of the final
+ # munging process.
+ #
+ def <<(reader_or_munge_or_writer)
+ case reader_or_munge_or_writer
+ when Reader
+ @readers << reader_or_munge_or_writer
+ when Munge
+ @munges << reader_or_munge_or_writer
+ when Writer
+ @writers << reader_or_munge_or_writer
+ else
+ fail "You must add a Reader, Munge, or Writer."
+ end
+ self # for chaining
+ end
+
+ #
+ # Constructs the final object tree for the munging process using all
+ # gathered Readers, Munges, and Writers.
+ #
+ # All Readers will be registered, in the order they are added, on the first
+ # Munge added. All Writers will be registered, in the order they are added,
+ # on the last Munge added. All Munges added are chained such that the first
+ # Munge added will be at the top of the chain and last Munge added will be
+ # at the bottom. The last Munge is returned so you can call run() on it to
+ # start the process.
+ #
+ # Adding a Munge is optional and a simple pass-through Munge will be created
+ # and returned (with all Readers and Writers attached), if none is added.
+ #
+ def build
+ fail "You need at least one Reader to build a Job." if @readers.empty?
+ fail "You need at least one Writer to build a Job." if @writers.empty?
+ if @munges.empty?
+ first_munge = last_munge = Munge.new do |m|
+ m.munge { |_, value| value }
+ end
+ else
+ first_munge = last_munge = @munges.pop
+ while munge_as_reader = @munges.pop
+ first_munge.add_reader(munge_as_reader)
+ first_munge = munge_as_reader
+ end
+ end
+ @readers.each do |reader|
+ first_munge.add_reader(reader)
+ end
+ @writers.each do |writer|
+ last_munge.add_writer(writer)
+ end
+ last_munge
+ end
+ end
+end
View
22 test/helper.rb
@@ -7,11 +7,31 @@ class MiniTest::Unit::TestCase
private
#######
- def reader(&init)
+ def reader(*inputs, &init)
+ unless inputs.empty?
+ return reader do |r|
+ r.read { inputs.shift }
+ end
+ end
@reader = Mungr::Reader.new(&init)
end
+ def munger(&init)
+ @munge = Mungr::Munge.new(&init)
+ end
+
+ def munge(&transform)
+ munger do |m|
+ m.munge { |_, *data| transform[*data] }
+ end
+ end
+
def writer(&init)
+ if init.nil?
+ return writer do |w|
+ w.write { }
+ end
+ end
@writer = Mungr::Writer.new(&init)
end
end
View
91 test/test_job.rb
@@ -0,0 +1,91 @@
+# encoding: UTF-8
+
+require "helper"
+
+require "mungr/job"
+
+class TestJob < MiniTest::Unit::TestCase
+ def setup
+ reset_job
+
+ @small_num_reader = reader(1, 2, 3)
+ @big_num_reader = reader(10, 20, 30)
+ @add_one_munge = munge { |*nums| nums.map { |n| n + 1 } }
+ @double_munge = munge { |*nums| nums.map { |n| n * 2 } }
+ @gather_writer = writer do |w|
+ w.prepare { @gathered = Array.new }
+ w.write { |gathered, *nums| gathered << nums }
+ end
+ @void_writer = writer
+
+ [ @small_num_reader, @big_num_reader,
+ @add_one_munge,
+ @double_munge,
+ @gather_writer, @void_writer ].each do |reader_or_munge_or_writer|
+ @job << reader_or_munge_or_writer
+ end
+ end
+
+ ####################
+ ### Requirements ###
+ ####################
+
+ def test_a_reader_is_required
+ reset_job
+ assert_raises(RuntimeError) do
+ @job.build
+ end
+ end
+
+ def test_a_writer_is_required
+ reset_job
+ @job << @small_num_reader
+ assert_raises(RuntimeError) do
+ @job.build
+ end
+ end
+
+ def test_a_munge_is_optional_with_a_pass_through_default
+ reset_job
+ @job << reader(:pass_through)
+ @job << @void_writer
+ munge = @job.build
+ assert_instance_of(Mungr::Munge, munge)
+ assert_equal(:pass_through, munge.munge)
+ end
+
+ #################
+ ### Interface ###
+ #################
+
+ def test_cannot_add_a_non_reader_non_munge_non_writer_object
+ assert_raises(RuntimeError) do
+ @job << :bad_object
+ end
+ end
+
+ def test_adds_can_be_chained
+ assert_same(@job, @job << @void_writer)
+ end
+
+ #################
+ ### Structure ###
+ #################
+
+ def test_the_last_munge_is_returned
+ assert_same(@double_munge, @job.build)
+ end
+
+ def test_readers_feed_chained_mungers_which_feed_writers
+ @job.build.run
+ assert_equal([[4, 22], [6, 42], [8, 62]], @gathered) # verify final results
+ end
+
+ #######
+ private
+ #######
+
+ def reset_job
+ @job = Mungr::Job.new
+ end
+end
View
53 test/test_munge.rb
@@ -11,7 +11,7 @@ class TestMunge < MiniTest::Unit::TestCase
def test_a_munge_is_prepared_before_the_first_munge
order = Array.new
- munge do |m|
+ munger do |m|
m.prepare { order << :prepared }
m.munge do
order << :munge
@@ -30,7 +30,7 @@ def test_exhausting_all_readers_sets_finished
[[1, 2, 3]],
[[1], [1, 2, 3], [1, 2]] ].each do |test_readers|
order = Array.new
- munge do |m|
+ munger do |m|
m.munge do
order << :munge
end
@@ -58,7 +58,7 @@ def test_exhausting_all_readers_sets_finished
def test_any_value_returned_from_prepare_is_forwarded_to_munge_and_finish
object = Object.new
calls = Array.new
- munge do |m|
+ munger do |m|
m.prepare { object }
m.munge do |context, _|
calls << context
@@ -78,7 +78,7 @@ def test_any_value_returned_from_prepare_is_forwarded_to_munge_and_finish
def test_add_reader_associates_input_sources_for_the_munge
args = Array.new
- munge do |m|
+ munger do |m|
m.munge do |_, i, l|
args << [i, l]
end
@@ -90,7 +90,7 @@ def test_add_reader_associates_input_sources_for_the_munge
end
def test_adding_a_munge_after_a_reader_is_an_error
- munge
+ munger
add_reader # add a normal Reader
assert_raises(RuntimeError) do
@munge.add_reader(Mungr::Munge.new) # add a Munge
@@ -98,7 +98,7 @@ def test_adding_a_munge_after_a_reader_is_an_error
end
def test_adding_any_kind_of_reader_after_a_munge_is_an_error
- munge
+ munger
@munge.add_reader(Mungr::Munge.new) # add a Munge
assert_raises(RuntimeError) do
@munge.add_reader(Mungr::Munge.new) # can't add another
@@ -114,7 +114,7 @@ def test_adding_any_kind_of_reader_after_a_munge_is_an_error
def test_add_writer_associates_output_sources_for_the_munge
args = Array.new
- munge do |m|
+ munger do |m|
m.munge do |_, i|
i
end
@@ -137,7 +137,7 @@ def test_add_writer_associates_output_sources_for_the_munge
end
def test_a_munge_with_a_writer_cannot_be_used_as_a_reader
- chain = munge
+ chain = munger
add_writer # can no longer be used as a Reader
munge
assert_raises(RuntimeError) do
@@ -152,7 +152,7 @@ def test_a_munge_with_a_writer_cannot_be_used_as_a_reader
def test_calling_munge_a_with_block_sets_the_code_and_further_calls_run_it
data = (1..3).to_a
written = Array.new
- munge do |m|
+ munger do |m|
m.munge do |_, i|
i * 2
end
@@ -169,7 +169,7 @@ def test_calling_munge_a_with_block_sets_the_code_and_further_calls_run_it
def test_each_input_is_passed_as_an_argument_to_munge
args = Array.new
- munge do |m|
+ munger do |m|
m.munge do |_, i, l|
args << [i, l]
end
@@ -182,7 +182,7 @@ def test_each_input_is_passed_as_an_argument_to_munge
def test_each_value_returned_from_munge_is_passed_as_an_argument_to_write
args = Array.new
- munge do |m|
+ munger do |m|
m.munge do |_, i|
[i, i * 2, i * 3]
end
@@ -199,7 +199,7 @@ def test_each_value_returned_from_munge_is_passed_as_an_argument_to_write
def test_when_readers_are_exhausted_no_more_munges_or_writes_are_made
calls = Array.new
- munge do |m|
+ munger do |m|
m.munge do
calls << :munge
end
@@ -218,7 +218,7 @@ def test_when_readers_are_exhausted_no_more_munges_or_writes_are_made
def test_prepare_is_called_once_before_the_first_munge
calls = Array.new
- munge do |m|
+ munger do |m|
m.prepare { calls << :prepare }
m.munge do
calls << :munge
@@ -231,7 +231,7 @@ def test_prepare_is_called_once_before_the_first_munge
def test_finish_is_called_once_when_the_input_is_exhausted
calls = Array.new
- munge do |m|
+ munger do |m|
m.munge do
calls << :munge
end
@@ -245,7 +245,7 @@ def test_finish_is_called_once_when_the_input_is_exhausted
end
def test_finish_is_forwarded_to_all_writers
- munge do |m|
+ munger do |m|
m.munge do
# do nothing: just defining some munge code
end
@@ -277,13 +277,13 @@ def test_finish_is_forwarded_to_all_writers
def test_a_munge_can_read_form_another_munge
args = Array.new
- chain = munge do |m|
+ chain = munger do |m|
m.munge do |_, i|
i * 2
end
end
add_reader(1, 2, 3)
- munge do |m|
+ munger do |m|
m.munge do |_, i2|
args << i2
end
@@ -295,13 +295,13 @@ def test_a_munge_can_read_form_another_munge
def test_a_munge_can_be_a_multireader_source_form_another_munge
args = Array.new
- chain = munge do |m|
+ chain = munger do |m|
m.munge do |_, i|
[i, i * 2, i * 3]
end
end
add_reader(1, 2, 3)
- munge do |m|
+ munger do |m|
m.munge do |_, i, i2, i3|
args << [i, i2, i3]
end
@@ -314,14 +314,14 @@ def test_a_munge_can_be_a_multireader_source_form_another_munge
def test_finish_is_forwarded_through_chained_munges
written = Array.new
chain = Array.new
- chain << munge do |m|
+ chain << munger do |m|
m.munge do |_, i, l|
[i, i * 2, l]
end
end
chain << add_reader(1, 2, 3).last
chain << add_reader(:a, :b).last
- chain << munge do |m|
+ chain << munger do |m|
m.munge do |_, i, i2, l|
"#{l}: #{i} * 2 = #{i2}"
end
@@ -341,17 +341,10 @@ def test_finish_is_forwarded_through_chained_munges
#######
private
#######
-
- def munge(&init)
- @munge = Mungr::Munge.new(&init)
- end
-
+
def add_reader(*inputs, &init)
(@readers ||= Array.new) <<
- ( inputs.empty? ? reader(&init) :
- reader { |r| r.read { inputs.shift } } ).tap { |r|
- @munge.add_reader(r)
- }
+ reader(*inputs, &init).tap { |r| @munge.add_reader(r) }
end
def add_writer(&init)
Please sign in to comment.
Something went wrong with that request. Please try again.