/
hdp-stream2
executable file
·39 lines (32 loc) · 1.08 KB
/
hdp-stream2
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
#!/usr/bin/env ruby
require 'wukong'
# Example usage:
#
# ~/ics/wukong/bin/hdp-stream2 input_path1,input_path2 output_path \
# "`which cuttab` 2,3,7" "`which uniq` -c" 1 3 -jobconf mapred.reduce.tasks=23
# options = Settings[:runner_defaults].dup
# cmdline_opts = Hash.zip(
# [ :input_file, :output_file,
# :map_command, :reduce_command,
# :partition_fields, :sort_fields],
# ARGV.map{|s| s.blank? ? nil : s }
# )
# argvs = ARGV.slice!(0..5) ;
# ARGV.unshift cmdline_opts[:input_file];
# ARGV.unshift cmdline_opts[:output_file]
# p [argvs, ARGV]
#
# # cmdline_opts[:map_command] = `which cat`.chomp if cmdline_opts[:map_command].blank?
# # cmdline_opts[:reduce_command] = nil if cmdline_opts[:reduce_command].blank?
# cmdline_opts[:dry_run] = true
# cmdline_opts[:run] = true
#p cmdline_opts, Settings[:runner_defaults]
# Go script go!
runner = Wukong::Script.new(
nil, # use mapper_command
nil, # use reducer_command
:run => true
)
# runner.options.merge cmdline_opts
runner.options[:reuse_jvms] = true if runner.options[:reuse_jvms].blank?
runner.run