Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

First commit of streaming example

  • Loading branch information...
commit f82e3bea407d9f6c929b6fef89b2a265c283ac3b 1 parent 4afbb65
@NathanNeff authored
View
BIN  examples/streaming/.mapper.pl.swp
Binary file not shown
View
BIN  examples/streaming/.run_streaming.sh.swp
Binary file not shown
View
BIN  examples/streaming/.view_output.sh.swp
Binary file not shown
View
9 examples/streaming/mapper.pl
@@ -0,0 +1,9 @@
+#!/usr/bin/env perl
+# Identity mapper
+use strict;
+use warnings;
+
+while (my $value = <STDIN>) {
+ chomp($value);
+ print "$value\n";
+}
View
32 examples/streaming/run_streaming.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+INPUT_DIR=input/streaming
+FILE_CONTENTS=this-script-contents
+DIR_CONTENTS=this-dir-contents
+OUTPUT_DIR=output/streaming
+
+MAPPER=mapper.pl
+REDUCER=mapper.pl
+
+function create_input() {
+ hadoop fs -rmr $OUTPUT_DIR
+ cat $0 > $FILE_CONTENTS
+ ls -l > $DIR_CONTENTS
+ hadoop fs -rmr $INPUT_DIR
+ hadoop fs -mkdir $INPUT_DIR
+ hadoop fs -put $FILE_CONTENTS $INPUT_DIR
+ hadoop fs -put $DIR_CONTENTS $INPUT_DIR
+ rm $FILE_CONTENTS $DIR_CONTENTS
+}
+
+create_input
+
+hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-streaming*jar \
+-mapper $MAPPER \
+-file $MAPPER \
+-reducer $MAPPER \
+-input $INPUT_DIR \
+-output $OUTPUT_DIR
+
+
+hadoop fs -ls $OUTPUT_DIR
+hadoop fs -cat $OUTPUT_DIR/part-00000
View
5 examples/streaming/this-dir
@@ -0,0 +1,5 @@
+total 1
+-rw-r--r-- 1 501 games 116 Jun 6 11:31 mapper.pl
+-rwxr-xr-x 1 501 games 587 Jun 6 11:33 run_streaming.sh
+-rw-r--r-- 1 501 games 0 Jun 6 11:33 this-dir
+-rw-r--r-- 1 501 games 587 Jun 6 11:33 this-script
View
5 examples/streaming/view_output.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+OUTPUT_DIR=output/streaming
+
+hadoop fs -ls $OUTPUT_DIR
+hadoop fs -cat $OUTPUT_DIR/part-00000
Please sign in to comment.
Something went wrong with that request. Please try again.