Skip to content
Browse files

Merge branch 'master' of github.com:Ganglion/swineherd

  • Loading branch information...
2 parents 08e9508 + 6b61737 commit b36a7cbd7883480d1b1c88485463c8903e03ded6 @thedatachef thedatachef committed Apr 18, 2011
Showing with 17 additions and 2 deletions.
  1. +1 −1 bin/hdp-tree
  2. +15 −0 lib/swineherd/filesystem/hadoopfilesystem.rb
  3. +1 −1 lib/swineherd/script/wukong_script.rb
View
2 bin/hdp-tree
@@ -22,5 +22,5 @@ def lr path
end
-tree = {path => lr(path)}.to_yaml
+tree = {File.dirname(path) => lr(path)}.to_yaml
puts tree
View
15 lib/swineherd/filesystem/hadoopfilesystem.rb
@@ -101,6 +101,21 @@ def stream input, output
end
#
+ # BZIP
+ #
+ def bzip input, output
+ system("#{@hadoop_home}/bin/hadoop \\
+ jar #{@hadoop_home}/contrib/streaming/hadoop-*streaming*.jar \\
+ -D mapred.output.compress=true \\
+ -D mapred.output.compression.codec=org.apache.hadoop.io.compress.BZip2Codec \\
+ -D mapred.reduce.tasks=1 \\
+ -mapper \"/bin/cat\" \\
+ -reducer \"/bin/cat\" \\
+ -input \"#{input}\" \\
+ -output \"#{output}\"")
+ end
+
+ #
# Copy hdfs file to local filesystem
#
def copy_to_local srcfile, dstfile
View
2 lib/swineherd/script/wukong_script.rb
@@ -21,7 +21,7 @@ def cmd
end
def local_cmd
- inputs = input.map{|path| path += "/*"}.join(',')
+ inputs = input.map{|path| path += File.directory?(path) ? "/*" : ""}.join(',')
"ruby #{script} #{wukong_args(@options)} --run=local #{inputs} #{output.join(',')}"
end

0 comments on commit b36a7cb

Please sign in to comment.
Something went wrong with that request. Please try again.