Skip to content
Browse files

there IS a difference between what the mapper and reducer commandline…

…s in local mode and reduce mode look like, seriously
  • Loading branch information...
1 parent 70b2158 commit 80526339ed7680c84a73dcdb8ed47a59e3b9d282 @thedatachef thedatachef committed Mar 4, 2011
Showing with 16 additions and 8 deletions.
  1. +14 −6 lib/wukong/script.rb
  2. +2 −2 lib/wukong/script/hadoop_command.rb
View
20 lib/wukong/script.rb
@@ -174,10 +174,14 @@ def run_mode
# In hadoop mode, this is given to the hadoop streaming command.
# In local mode, it's given to the system() call
#
- def mapper_commandline
+ def mapper_commandline(run_option=:local)
if mapper
- "#{ruby_interpreter_path} #{this_script_filename} --map " + non_wukong_params
- # "#{ruby_interpreter_path} #{File.basename(this_script_filename)} --map " + non_wukong_params
+ case run_option
+ when :local then
+ "#{ruby_interpreter_path} #{this_script_filename} --map " + non_wukong_params
+ when :hadoop then
+ "#{ruby_interpreter_path} #{File.basename(this_script_filename)} --map " + non_wukong_params
+ end
else
options[:map_command]
end
@@ -188,10 +192,14 @@ def mapper_commandline
# In hadoop mode, this is given to the hadoop streaming command.
# In local mode, it's given to the system() call
#
- def reducer_commandline
+ def reducer_commandline(run_option=:local)
if reducer
- "#{ruby_interpreter_path} #{this_script_filename} --reduce " + non_wukong_params
- # "#{ruby_interpreter_path} #{File.basename(this_script_filename)} --reduce " + non_wukong_params
+ case run_option
+ when :local then
+ "#{ruby_interpreter_path} #{this_script_filename} --reduce " + non_wukong_params
+ when :hadoop then
+ "#{ruby_interpreter_path} #{File.basename(this_script_filename)} --reduce " + non_wukong_params
+ end
else
options[:reduce_command]
end
View
4 lib/wukong/script/hadoop_command.rb
@@ -64,8 +64,8 @@ def execute_hadoop_workflow
hadoop_jobconf_options,
"-D mapred.job.name='#{job_name}'",
hadoop_other_args,
- "-mapper '#{mapper_commandline}'",
- "-reducer '#{reducer_commandline}'",
+ "-mapper '#{mapper_commandline(:hadoop)}'",
+ "-reducer '#{reducer_commandline(:hadoop)}'",
"-input '#{input_paths}'",
"-output '#{output_path}'",
"-file '#{this_script_filename}'",

0 comments on commit 8052633

Please sign in to comment.
Something went wrong with that request. Please try again.