new: dev: Added preliminary support for Tez(#10)

qubole · Jan 19, 2017 · 78d2cc8 · 78d2cc8
1 parent bf6bdee
commit 78d2cc8
Show file tree

Hide file tree

Showing 13 changed files with 1,140 additions and 3 deletions.
diff --git a/app-conf/AggregatorConf.xml b/app-conf/AggregatorConf.xml
@@ -33,6 +33,11 @@
     <applicationtype>mapreduce</applicationtype>
     <classname>com.linkedin.drelephant.mapreduce.MapReduceMetricsAggregator</classname>
   </aggregator>
+  <aggregator>
+    <applicationtype>tez</applicationtype>
+    <classname>com.linkedin.drelephant.tez.TezMetricsAggregator</classname>
+  </aggregator>
+
   <aggregator>
     <applicationtype>spark</applicationtype>
     <classname>org.apache.spark.SparkMetricsAggregator</classname>

diff --git a/app-conf/FetcherConf.xml b/app-conf/FetcherConf.xml
@@ -56,6 +56,10 @@
     </params>
   </fetcher>
   -->
+  <fetcher>
+    <applicationtype>tez</applicationtype>
+    <classname>com.linkedin.drelephant.tez.TezFetcher</classname>
+  </fetcher>
   <fetcher>
     <applicationtype>spark</applicationtype>
     <classname>org.apache.spark.deploy.history.SparkFSFetcher</classname>

diff --git a/app-conf/HeuristicConf.xml b/app-conf/HeuristicConf.xml
@@ -16,7 +16,19 @@
 -->
 <!-- Heuristics configurations, each heuristic will be loaded by a particular analyser -->
 <heuristics>
-
+
+  <!-- TEZ HEURISTICS -->
+  <heuristic>
+    <applicationtype>tez</applicationtype>
+    <heuristicname>Mapper Spill</heuristicname>
+    <classname>com.linkedin.drelephant.tez.heuristics.MapperSpillHeuristic</classname>
+    <viewname>views.html.help.mapreduce.helpMapperSpill</viewname>
+    <!--<params>
+      <spill_severity>2.01, 2.2, 2.5, 3</spill_severity>
+      <num_tasks_severity>50, 100, 500, 1000</num_tasks_severity>
+    </params>-->
+  </heuristic> 
+
   <!-- MAP-REDUCE HEURISTICS -->
 
   <heuristic>

diff --git a/app-conf/JobTypeConf.xml b/app-conf/JobTypeConf.xml
@@ -42,6 +42,12 @@
     <applicationtype>mapreduce</applicationtype>
     <conf>pig.script</conf>
   </jobType>
+  <jobType>
+    <name>Hive</name>
+    <applicationtype>tez</applicationtype>
+    <conf>hive.mapred.mode</conf>
+    <isDefault/>
+  </jobType>
   <jobType>
     <name>Hive</name>
     <applicationtype>mapreduce</applicationtype>