get stuff to run on yarn

Author: vangj

spark-cluster/master/README.md

@@ -0,0 +1,34 @@
+# Notes
+
+## Testing
+
+* [HDFS](http://localhost:9870)
+* [YARN](http://localhost:8088)
+* [Spark](http://localhost:8080)
+* [Spark History](http://localhost:18080)
+
+```bash
+# test hadoop/spark services
+jps
+
+# test listening ports
+netstat -tulnp | grep LISTEN
+
+# test hadoop
+hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar pi 5 10
+
+# test yarn
+yarn jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar pi 1 50
+
+# test spark with yarn
+$SPARK_HOME/bin/spark-submit --class org.apache.spark.examples.SparkPi \
+    --master yarn \
+    $SPARK_HOME/examples/jars/spark-examples*.jar \
+    100
+
+# test spark standalone
+$SPARK_HOME/bin/spark-submit --class org.apache.spark.examples.SparkPi \
+    --master spark://localhost:7077 \
+    $SPARK_HOME/examples/jars/spark-examples*.jar \
+    100
+```

spark-cluster/master/ubuntu/usr/local/hadoop/etc/hadoop/mapred-site.xml

@@ -17,5 +17,23 @@
 <!-- Put site-specific property overrides in this file. -->
 
 <configuration>
+  <property>
+    <name>mapreduce.framework.name</name>
+    <value>yarn</value>
+  </property>
+  
+  <property>
+    <name>yarn.app.mapreduce.am.env</name>
+    <value>HADOOP_MAPRED_HOME=/usr/local/hadoop</value>
+  </property>
 
+  <property>
+    <name>mapreduce.map.env</name>
+    <value>HADOOP_MAPRED_HOME=/usr/local/hadoop</value>
+  </property>
+
+  <property>
+    <name>mapreduce.reduce.env</name>
+    <value>HADOOP_MAPRED_HOME=/usr/local/hadoop</value>
+  </property>
 </configuration>