From 807df966eb6b006d41a87e07f0d8609170034cd9 Mon Sep 17 00:00:00 2001 From: fegaras Date: Sat, 13 Feb 2016 15:15:32 -0600 Subject: [PATCH] [MRQL-83] Make Hadoop 2.* (Yarn) the default and update pom files --- bsp/pom.xml | 1 + .../java/org/apache/mrql/BSPEvaluator.gen | 4 ++-- conf/mrql-env.sh | 22 +++++++++---------- flink/pom.xml | 1 + .../java/org/apache/mrql/FlinkEvaluator.gen | 4 ++-- mapreduce/pom.xml | 1 + .../org/apache/mrql/MapReduceEvaluator.gen | 6 ++--- pom.xml | 18 ++++++++++----- spark/pom.xml | 7 ++++++ 9 files changed, 39 insertions(+), 25 deletions(-) diff --git a/bsp/pom.xml b/bsp/pom.xml index a9b940a..3a3073d 100644 --- a/bsp/pom.xml +++ b/bsp/pom.xml @@ -58,6 +58,7 @@ junit junit + 4.11 test diff --git a/bsp/src/main/java/org/apache/mrql/BSPEvaluator.gen b/bsp/src/main/java/org/apache/mrql/BSPEvaluator.gen index 6f6a4bf..37da613 100644 --- a/bsp/src/main/java/org/apache/mrql/BSPEvaluator.gen +++ b/bsp/src/main/java/org/apache/mrql/BSPEvaluator.gen @@ -36,12 +36,12 @@ final public class BSPEvaluator extends Evaluator { conf.set("bsp.master.address","local"); conf.set("hama.zookeeper.quorum","localhost"); conf.setInt("bsp.local.tasks.maximum",Config.nodes); - conf.set("fs.default.name","file:///"); + conf.set("fs.defaultFS","file:///"); } else { conf.set("bsp.master.address",System.getenv("BSP_MASTER_ADDRESS")); conf.set("hama.zookeeper.quorum",System.getenv("HAMA_ZOOKEEPER_QUORUM")); conf.setInt("bsp.local.tasks.maximum",Config.nodes); - conf.set("fs.default.name",System.getenv("FS_DEFAULT_NAME")); + conf.set("fs.defaultFS",System.getenv("FS_DEFAULT_NAME")); } } diff --git a/conf/mrql-env.sh b/conf/mrql-env.sh index fac62b1..135d96c 100644 --- a/conf/mrql-env.sh +++ b/conf/mrql-env.sh @@ -22,11 +22,11 @@ # # To rebuild Apache MRQL from sources: # -# build MRQL on Hadoop 1.x: -# mvn -Dhadoop.version=1.0.3 clean install -# # build MRQL on Hadoop 2.x (yarn): -# mvn -Dyarn -Dyarn.version=2.2.0 -Dhadoop.version=1.2.1 clean install +# mvn -Dyarn.version=2.2.0 clean install +# +# build MRQL on Hadoop 1.x: +# mvn -Dhadoop1 -Dhadoop.version=1.0.3 clean install # #-------------------------------------------------------------------------------- # @@ -48,7 +48,7 @@ JLINE_JAR=${HOME}/.m2/repository/jline/jline/1.0/jline-1.0.jar # Required: Hadoop configuration. Supports versions 1.x and 2.x (YARN) -HADOOP_VERSION=2.6.0 +HADOOP_VERSION=2.7.1 # The Hadoop installation directory HADOOP_HOME=${HOME}/hadoop-${HADOOP_VERSION} # The Hadoop configuration directory (where core-site.xml is) @@ -59,8 +59,8 @@ MAPRED_JOB_TRACKER=localhost:9001 FS_DEFAULT_NAME=hdfs://localhost:9000/ -# Optional: Hama configuration. Supports versions 0.6.2, 0.6.3, and 0.6.4 -HAMA_VERSION=0.6.4 +# Optional: Hama configuration. Supports versions 0.6.2, 0.6.3, 0.6.4, and 0.7.0 +HAMA_VERSION=0.7.0 # The Hama installation directory HAMA_HOME=${HOME}/hama-${HAMA_VERSION} # The Hama configuration directory @@ -72,12 +72,12 @@ HAMA_ZOOKEEPER_QUORUM=localhost BSP_SPLIT_INPUT= -# Optional: Spark configuration. Supports versions 1.0.0, 1.0.2, 1.1.0, 1.1.1, 1.2.0, 1.3.0, and 1.3.1 +# Optional: Spark configuration. Supports versions 1.0.0, 1.0.2, 1.1.0, 1.1.1, 1.2.0, 1.3.0, 1.3.1, and 1.6.0 # (Spark versions 0.8.1, 0.9.0, and 0.9.1 are supported by MRQL 0.9.0) # You may use the Spark prebuilts bin-hadoop1 or bin-hadoop2 (Yarn) # For distributed mode, give write permission to /tmp: hadoop fs -chmod -R 777 /tmp # Tested in local, standalone deploy, and Yarn modes -SPARK_HOME=${HOME}/spark-1.3.0-bin-hadoop2.4 +SPARK_HOME=${HOME}/spark-1.6.0-bin-hadoop2.6 # URI of the Spark master node: # to run Spark on Standalone Mode, set it to spark://`hostname`:7077 # to run Spark on a YARN cluster, set it to "yarn-client" @@ -93,8 +93,8 @@ SPARK_EXECUTOR_CORES=1 SPARK_EXECUTOR_MEMORY=1G -# Optional: Flink configuration. Supports version 0.10.1 -FLINK_VERSION=0.10.1 +# Optional: Flink configuration. Supports versions 0.10.1 and 0.10.2 +FLINK_VERSION=0.10.2 # Flink installation directory FLINK_HOME=${HOME}/flink-${FLINK_VERSION} # number of slots per TaskManager (typically, the number of CPUs per machine) diff --git a/flink/pom.xml b/flink/pom.xml index 3b01456..9243682 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -68,6 +68,7 @@ junit junit + 4.11 test diff --git a/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen b/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen index 83e80c2..1f2637f 100644 --- a/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen +++ b/flink/src/main/java/org/apache/mrql/FlinkEvaluator.gen @@ -68,7 +68,7 @@ public class FlinkEvaluator extends Evaluator implements Serializable { // curently, the compiler doesn't work in local mode Config.compile_functional_arguments = false; fs_default_name = "file:///"; - Plan.conf.set("fs.default.name",fs_default_name); + Plan.conf.set("fs.defaultFS",fs_default_name); } else if (Config.distributed_mode) { String master_node = System.getenv("FLINK_MASTER"); if (master_node == null) @@ -82,7 +82,7 @@ public class FlinkEvaluator extends Evaluator implements Serializable { }; fs_default_name = System.getenv("FS_DEFAULT_NAME"); data_source_dir_name = absolute_path("tmp/data_source_dir.txt"); - Plan.conf.set("fs.default.name",fs_default_name); + Plan.conf.set("fs.defaultFS",fs_default_name); } } diff --git a/mapreduce/pom.xml b/mapreduce/pom.xml index a54799c..448bf67 100644 --- a/mapreduce/pom.xml +++ b/mapreduce/pom.xml @@ -53,6 +53,7 @@ junit junit + 4.11 test diff --git a/mapreduce/src/main/java/org/apache/mrql/MapReduceEvaluator.gen b/mapreduce/src/main/java/org/apache/mrql/MapReduceEvaluator.gen index 1fdc5cb..ac6e93a 100644 --- a/mapreduce/src/main/java/org/apache/mrql/MapReduceEvaluator.gen +++ b/mapreduce/src/main/java/org/apache/mrql/MapReduceEvaluator.gen @@ -35,12 +35,10 @@ final public class MapReduceEvaluator extends Evaluator { if (Config.local_mode) { conf.set("mapred.job.tracker","local"); conf.set("mapreduce.framework.name","local"); - conf.set("fs.default.name","file:///"); - //conf.set("fs.defaultFS","file:///"); // doesn't work for hadoop 1.* + conf.set("fs.defaultFS","file:///"); // doesn't work for hadoop 1.* } else { conf.set("mapred.job.tracker",System.getenv("MAPRED_JOB_TRACKER")); - conf.set("fs.default.name",System.getenv("FS_DEFAULT_NAME")); - //conf.set("fs.defaultFS",System.getenv("FS_DEFAULT_NAME")); // doesn't work for hadoop 1.* + conf.set("fs.defaultFS",System.getenv("FS_DEFAULT_NAME")); // doesn't work for hadoop 1.* } } diff --git a/pom.xml b/pom.xml index f1f0029..568e891 100644 --- a/pom.xml +++ b/pom.xml @@ -45,11 +45,11 @@ 1.2.1 - 2.6.0 - 0.6.4 - 1.3.1 + 2.7.1 + 0.7.0 + 1.6.0 2.10 - 0.10.1 + 0.10.2 true @@ -79,6 +79,12 @@ jline 1.0 + + junit + junit + 4.11 + test + @@ -210,7 +216,7 @@ hadoop1 - !yarn + hadoop1 @@ -230,7 +236,7 @@ yarn - yarn + !hadoop1 diff --git a/spark/pom.xml b/spark/pom.xml index 4c7ce91..e36b49e 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -63,8 +63,15 @@ junit junit + 4.11 test + + + commons-io + commons-io + 2.4 +