Adding Hadoop Yarn Common into the openctest framework

core/add_project.sh

@@ -11,6 +11,8 @@ function setup_hadoop() {
     mvn clean install -DskipTests
     cd $home_dir/hadoop-hdfs-project/hadoop-hdfs
     mvn package -DskipTests
+    cd $home_dir/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common
+    mvn package -DskipTests
 }
 
 function setup_hbase() {

core/ctest_const.py

@@ -12,6 +12,7 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+HYARNCOMMON = "hadoop-yarn-common"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
@@ -24,6 +25,7 @@
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    HYARNCOMMON: CTEST_HADOOP_DIR,
 }
 
 
@@ -34,6 +36,7 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    HYARNCOMMON: "hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common"
 }
 
 
@@ -58,6 +61,7 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    HYARNCOMMON: [os.path.join(CTEST_HADOOP_DIR, MODULE_SUBDIR[HYARNCOMMON], SUREFIRE_SUBDIR)],
 }
 
 # default or deprecate conf path
@@ -74,7 +78,8 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    HYARNCOMMON: os.path.join(DEFAULT_CONF_DIR, HYARNCOMMON + "-default.tsv"),
 }
 
 
@@ -96,6 +101,9 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
+    ],
+    HYARNCOMMON: [
+        os.path.join(CTEST_HADOOP_DIR, "hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/classes/yarn-common-ctest.xml")
     ]
 }
 

core/generate_ctest/inject.py

@@ -21,7 +21,7 @@ def inject_config(param_value_pairs):
             for p, v in param_value_pairs.items():
                 file.write(p + "=" + v + "\n")
             file.close()
-    elif project in [HCOMMON, HDFS, HBASE]:
+    elif project in [HCOMMON, HDFS, HBASE, HYARNCOMMON]:
         conf = ET.Element("configuration")
         for p, v in param_value_pairs.items():
             prop = ET.SubElement(conf, "property")
@@ -46,7 +46,7 @@ def clean_conf_file(project):
             file = open(inject_path, "w")
             file.write("\n")
             file.close()
-    elif project in [HCOMMON, HDFS, HBASE]:
+    elif project in [HCOMMON, HDFS, HBASE, HYARNCOMMON]:
         conf = ET.Element("configuration")
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "wb")

core/generate_ctest/main.py

@@ -33,7 +33,7 @@ def test_value_pair(test_input):
         mt_file = open(os.path.join(GENCTEST_TR_DIR, project, MT_FILE.format(id=param)), "w")
 
         associated_tests = mapping[param] if param in mapping else []
-        if len(mapping[param]) != 0:
+        if param in mapping and len(mapping[param]) != 0:
             for value in values:
                 tr = run_test_seperate(param, value, associated_tests)
 

core/generate_ctest/program_input.py

@@ -4,11 +4,11 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "hadoop-yarn-common", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-yarn-common.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "hadoop-yarn-common.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test

core/generate_value/value_generation.py

@@ -25,6 +25,9 @@ def read_tsv(module):
     if module == "zookeeper-server":
         assert len(params) == 32
         return 32
+    elif module == "hadoop-yarn-common":
+        assert len(params) == 574
+        return 574
     else:
         assert len(params) == 90
         return 90
@@ -105,6 +108,8 @@ def print_params(module):
     f = open(module + output, "w")
     if module == "zookeeper-server":
         assert len(params) == 32
+    elif module == "hadoop-yarn-common":
+        assert len(params) == 574
     else:
         assert len(params) >= 90
     for param in params:

core/identify_param/add_project.sh

@@ -11,6 +11,8 @@ function setup_hadoop() {
     mvn clean install -DskipTests
     cd $home_dir/hadoop-hdfs-project/hadoop-hdfs
     mvn package -DskipTests
+    cd $home_dir/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common
+    mvn package -DskipTests
 }
 
 function setup_hbase() {

core/identify_param/constant.py

@@ -12,15 +12,17 @@
     "hadoop-common": CTEST_HADOOP_DIR,
     "hadoop-hdfs": CTEST_HADOOP_DIR,
     "hbase-server": CTEST_HBASE_DIR,
-    "alluxio-core": CTEST_ALLUXIO_DIR
+    "alluxio-core": CTEST_ALLUXIO_DIR,
+    "hadoop-yarn-common": CTEST_HADOOP_DIR
 }
 
 SRC_SUBDIR = {
     "hadoop-common": "hadoop-common-project/hadoop-common",
     "hadoop-hdfs": "hadoop-hdfs-project/hadoop-hdfs",
     "hbase-server": "hbase-server",
     "zookeeper-server": "zookeeper-server",
-    "alluxio-core": "core"
+    "alluxio-core": "core",
+    "hadoop-yarn-common": "hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common"
 }
 
 MVN_TEST_PATH = {
@@ -29,14 +31,16 @@
     "hbase-server": os.path.join(CTEST_HBASE_DIR, SRC_SUBDIR["hbase-server"]),
     "zookeeper-server": os.path.join(CTEST_ZOOKEEPER_DIR, SRC_SUBDIR["zookeeper-server"]),
     "alluxio-core": os.path.join(CTEST_ALLUXIO_DIR, SRC_SUBDIR["alluxio-core"]),
+    "hadoop-yarn-common": os.path.join(CTEST_HADOOP_DIR, SRC_SUBDIR["hadoop-yarn-common"]),
 }
 
 LOCAL_CONF_PATH = {
     "hadoop-common": "results/hadoop-common/conf_params.txt",
     "hadoop-hdfs": "results/hadoop-hdfs/conf_params.txt",
     "hbase-server": "results/hbase-server/conf_params.txt",
     "zookeeper-server": "results/zookeeper-server/conf_params.txt",
-    "alluxio-core": "results/alluxio-core/conf_params.txt"
+    "alluxio-core": "results/alluxio-core/conf_params.txt",
+    "hadoop-yarn-common": "results/hadoop-yarn-common/conf_params.txt",
 }
 
 SUREFIRE_SUBDIR = "target/surefire-reports/*"
@@ -63,7 +67,10 @@
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/proxy", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/master", SUREFIRE_SUBDIR)
-    ]
+    ],
+    "hadoop-yarn-common": [
+        os.path.join(CTEST_HADOOP_DIR, SRC_SUBDIR["hadoop-yarn-common"], SUREFIRE_SUBDIR)
+    ],
 }
 
 LOCAL_SUREFIRE_SUFFIX = "surefire-reports/*"
@@ -83,5 +90,8 @@
     ],
     "alluxio-core": [
         os.path.join("surefire-reports/alluxio-core", LOCAL_SUREFIRE_SUFFIX)
-    ]
+    ],
+    "hadoop-yarn-common": [
+        os.path.join("surefire-reports/yarn/hadoop-yarn", LOCAL_SUREFIRE_SUFFIX)
+    ],
 }

core/identify_param/identify_param.sh

@@ -12,7 +12,7 @@ function main() {
       usage
     else
         case $project in
-            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core) python3 runner.py $project; python3 collector.py $project ;;
+            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core | hadoop-yarn-common) python3 runner.py $project; python3 collector.py $project ;;
             -h | --help) usage ;;
             *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server and alluxio-core." ;;
         esac