Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

resolved conflict in script with File.basename on the script filename…

… -- what's up with that?
  • Loading branch information...
commit 86183f32c785a8a687f92346d167560c5e0ec6b1 2 parents 947156b + fd06433
Philip (flip) Kromer authored
View
29 bin/hdp-bzip
@@ -2,27 +2,22 @@
HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
-OUTPUT="$1" ; shift
+input_file=${1} ; shift
+output_file=${1} ; shift
-INPUTS=''
-for foo in $@; do
- INPUTS="$INPUTS -input $foo\
-"
-done
+if [ "$output_file" == "" ] ; then echo "$0 input_file output_file" ; exit ; fi
-echo "Removing output directory $OUTPUT"
-hadoop fs -rmr $OUTPUT
+HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
cmd="${HADOOP_HOME}/bin/hadoop \
- jar ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar \
- -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
- -jobconf mapred.output.compress=true \
- -jobconf mapred.output.compression.codec=org.apache.hadoop.io.compress.BZip2Codec \
- -jobconf mapred.reduce.tasks=1 \
- -mapper \"/bin/cat\" \
- -reducer \"/bin/cat\" \
- $INPUTS
- -output $OUTPUT \
+ jar ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar \
+ -Dmapred.output.compress=true \
+ -Dmapred.output.compression.codec=org.apache.hadoop.io.compress.BZip2Codec \
+ -Dmapred.reduce.tasks=1 \
+ -mapper \"/bin/cat\" \
+ -reducer \"/bin/cat\" \
+ -input \"$input_file\" \
+ -output \"$output_file\" \
"
echo $cmd
$cmd
View
2  bin/hdp-kill-task
@@ -1,3 +1,3 @@
#!/usr/bin/env bash
-exec hadoop fs -kill-task "$1"
+exec hadoop job -kill-task "$1"
View
15 bin/hdp-stream
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-# hadoop dfs -rmr out/parsed-followers
input_file=${1} ; shift
output_file=${1} ; shift
@@ -13,18 +12,18 @@ if [ "$output_file" == "" ] ; then echo "$0 input_file output_file [mapper=/bin/
HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
cmd="${HADOOP_HOME}/bin/hadoop \
- jar ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar
+ jar ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar
+ $@
+ -D num.key.fields.for.partition=\"$partfields\"
+ -D stream.num.map.output.key.fields=\"$sortfields\"
+ -D stream.map.output.field.separator=\"'/t'\"
+ -D mapred.text.key.partitioner.options=\"-k1,$partfields\"
+ -D mapred.job.name=\"`basename $0`-$map_script-$input_file-$output_file\"
-partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner
- -jobconf "mapred.job.name=`basename $0`-$map_script-$input_file-$output_file"
- -jobconf num.key.fields.for.partition=\"$partfields\"
- -jobconf stream.num.map.output.key.fields=\"$sortfields\"
- -jobconf stream.map.output.field.separator=\"'/t'\"
- -jobconf mapred.text.key.partitioner.options=\"-k1,$partfields\"
-mapper \"$map_script\"
-reducer \"$reduce_script\"
-input \"$input_file\"
-output \"$output_file\"
- $@
"
echo "$cmd"
View
2  bin/hdp-stream-flat
@@ -16,7 +16,7 @@ HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
exec ${HADOOP_HOME}/bin/hadoop \
jar ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar \
"$@" \
- -jobconf "mapred.job.name=`basename $0`-$map_script-$input_file-$output_file" \
+ -D "mapred.job.name=`basename $0`-$map_script-$input_file-$output_file" \
-mapper "$map_script" \
-reducer "$reduce_script" \
-input "$input_file" \
View
4 lib/wukong/script.rb
@@ -176,6 +176,7 @@ def run_mode
def mapper_commandline
if mapper
"#{ruby_interpreter_path} #{this_script_filename} --map " + non_wukong_params
+ # "#{ruby_interpreter_path} #{File.basename(this_script_filename)} --map " + non_wukong_params
else
options[:map_command]
end
@@ -188,7 +189,8 @@ def mapper_commandline
#
def reducer_commandline
if reducer
- "#{ruby_interpreter_path} #{this_script_filename} --reduce " + non_wukong_params
+ "#{ruby_interpreter_path} #{this_script_filename} --reduce " + non_wukong_params
+ # "#{ruby_interpreter_path} #{File.basename(this_script_filename)} --reduce " + non_wukong_params
else
options[:reduce_command]
end
View
1  lib/wukong/script/hadoop_command.rb
@@ -68,6 +68,7 @@ def execute_hadoop_workflow
"-reducer '#{reducer_commandline}'",
"-input '#{input_paths}'",
"-output '#{output_path}'",
+ "-file '#{this_script_filename}'",
hadoop_recycle_env,
].flatten.compact.join(" \t\\\n ")
Log.info " Launching hadoop!"
Please sign in to comment.
Something went wrong with that request. Please try again.