Permalink
Browse files

local mode should actually work now (at least for pig scripts and r s…

…cripts), still broken for wukong scripts since --run=local is broken
  • Loading branch information...
1 parent 089baf4 commit 57b23bd8d630c70805faa3a7ec6340230c0ed756 @thedatachef thedatachef committed Feb 18, 2011
@@ -30,17 +30,27 @@ def refresh!
def cmd
raise "Override this in subclass!"
end
+
+ #
+ # Override this in subclass to decide how script runs in 'local' mode
+ # Best practice is that it needs to be able to run on a laptop w/o
+ # hadoop.
+ #
+ def local_cmd
+ raise "Override this in subclass!"
+ end
+
#
# Default is to run with hadoop
#
def run mode=:hadoop
case mode
when :local then
localfs = FileSystem.get :file
- sh "#{cmd}" if localfs.check_paths(@output)
+ sh local_cmd if localfs.check_paths(@output)
when :hadoop then
hdfs = FileSystem.get :hdfs
- sh "#{cmd}" if hdfs.check_paths(@output)
+ sh cmd if hdfs.check_paths(@output)
end
end
@@ -11,6 +11,10 @@ def pig_args options
options.map{|opt,val| "-p #{opt.to_s.upcase}=#{val}" }.join(' ')
end
+ def local_cmd
+ "PIG_CLASSPATH=#{@pig_classpath} PIG_OPTS='#{@pig_options}' pig -x local #{pig_args(@options)} #{script}"
+ end
+
def cmd
"PIG_CLASSPATH=#{@pig_classpath} PIG_OPTS='#{@pig_options}' pig #{pig_args(@options)} #{script}"
end
@@ -1,8 +1,14 @@
module Swineherd::Script
class RScript
include Common
- def cmd
+
+ def local_cmd
"/usr/bin/Rscript --vanilla #{script}"
end
+
+ def cmd
+ local_cmd
+ end
+
end
end
@@ -18,5 +18,12 @@ def cmd
raise "No wukong input specified" if input.empty?
"#{ruby_interpreter_path} #{script} #{wukong_args(@options)} --run #{input.join(',')} #{output.join(',')}"
end
+
+ # FIXME: wukong's local mode doesn't work?
+ def local_cmd
+ inputs = input.map{|path| path += "/*"}.join(',')
+ "#{ruby_interpreter_path} #{script} #{wukong_args(@options)} --run=local #{inputs} #{output.join(',')}"
+ end
+
end
end

0 comments on commit 57b23bd

Please sign in to comment.