Permalink
Browse files

version bump to 0.0.2 to support wukong 3.0.0.pre3. also added specul…

…ative execution flags to hadoop
  • Loading branch information...
1 parent f8ebbfc commit b51794e4c167c7801eaabeba5981ef6914572dc0 @dhruvbansal dhruvbansal committed Dec 17, 2012
View
5 Gemfile
@@ -1,3 +1,8 @@
source :rubygems
gemspec
+
+group :development do
+ gem 'rake', '~> 0.9'
+ gem 'rspec', '~> 2'
+end
View
57 Gemfile.lock
@@ -0,0 +1,57 @@
+PATH
+ remote: .
+ specs:
+ wonderdog (0.0.1)
+ wukong (= 3.0.0.pre3)
+ wukong-hadoop (>= 0.0.2)
+
+GEM
+ remote: http://rubygems.org/
+ specs:
+ configliere (0.4.18)
+ highline (>= 1.5.2)
+ multi_json (>= 1.1)
+ diff-lcs (1.1.3)
+ eventmachine (1.0.0)
+ forgery (0.5.0)
+ gorillib (0.4.2)
+ configliere (>= 0.4.13)
+ json
+ multi_json (>= 1.1)
+ highline (1.6.15)
+ json (1.7.5)
+ log4r (1.1.10)
+ multi_json (1.5.0)
+ rake (0.9.6)
+ rspec (2.12.0)
+ rspec-core (~> 2.12.0)
+ rspec-expectations (~> 2.12.0)
+ rspec-mocks (~> 2.12.0)
+ rspec-core (2.12.2)
+ rspec-expectations (2.12.1)
+ diff-lcs (~> 1.1.3)
+ rspec-mocks (2.12.0)
+ uuidtools (2.1.3)
+ vayacondios-client (0.1.2)
+ configliere (>= 0.4.16)
+ gorillib (~> 0.4.2)
+ multi_json (~> 1.1)
+ wukong (3.0.0.pre3)
+ configliere (>= 0.4.18)
+ eventmachine
+ forgery
+ gorillib (>= 0.4.2)
+ log4r
+ multi_json (>= 1.3.6)
+ uuidtools
+ vayacondios-client (>= 0.1.2)
+ wukong-hadoop (0.0.2)
+ wukong (= 3.0.0.pre3)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ rake (~> 0.9)
+ rspec (~> 2)
+ wonderdog!
View
9 lib/wonderdog/hadoop_invocation_override.rb
@@ -98,8 +98,15 @@ def output_path
#
# @return [Array<String>]
def hadoop_jobconf_options
+ if reads_from_elasticsearch? || writes_to_elasticsearch?
+ settings[:map_speculative] = false if settings[:map_speculative].nil?
+ settings[:reduce_speculative] = false if settings[:reduce_speculative].nil?
+ end
+
super() + [].tap do |o|
- o << java_opt('es.config', settings[:es_config]) if (reads_from_elasticsearch? || writes_to_elasticsearch?)
+ if (reads_from_elasticsearch? || writes_to_elasticsearch?)
+ o << java_opt('es.config', settings[:es_config])
+ end
if reads_from_elasticsearch?
o << java_opt('elasticsearch.input.index', input_index.index)
View
3 lib/wonderdog/version.rb
@@ -1,3 +1,4 @@
module Wonderdog
- VERSION = '0.0.1'
+ # The currently running Wonderdog version
+ VERSION = '0.0.2'
end
View
16 spec/wonderdog/hadoop_invocation_override_spec.rb
@@ -32,6 +32,22 @@
end
end
end
+
+ context "setting speculative execution" do
+ context "when not given speculative options" do
+ context "and not interacting with Elasticsearch" do
+ it "doesn't add jars" do
+ no_es.hadoop_commandline.should_not match('speculative')
+ end
+ end
+ context "and reading from Elasticsearch" do
+ it "adds default jars it finds on the local filesystem" do
+ es_reader.hadoop_commandline.should match('-mapred.map.tasks.speculative.execution.*false')
+ es_reader.hadoop_commandline.should match('-mapred.reduce.tasks.speculative.execution.*false')
+ end
+ end
+ end
+ end
context "handling input and output paths, formats, and options when" do
View
3 wonderdog.gemspec
@@ -28,5 +28,6 @@ EOF
gem.test_files = gem.files.grep(/^spec/)
gem.require_paths = ['lib']
- gem.add_dependency('wukong', '3.0.0.pre2')
+ gem.add_dependency('wukong', '3.0.0.pre3')
+ gem.add_dependency('wukong-hadoop', '>= 0.0.2')
end

0 comments on commit b51794e

Please sign in to comment.