[ADAM-2145] Add Spark 2.4.3 and Scala 2.12 to Jenkins build

* Add Spark 2.4.3 and Scala 2.12 to Jenkins build.

* Fix for Jenkins failures.

* remove move_to_spark_2.sh check

* move back to scala 2.11

* move before git test

* still not in correct place

* Spark 2.4.3 needs special case for Scala 2.12

* export SPARK_HOME correctly

* download hadoop and add to classpath

* try adding explicit dependency on paranamer 2.8

* try specifying spark.executor.memory

* try PYSPARK_SUBMIT_ARGS in Makefile

* try again

* fix conf syntax

* move conf to jenkins-test

* remove references to avro 1.7.x

* install avro 1.8.x

adam-cli/src/main/scala/org/bdgenomics/adam/cli/FlagStat.scala

@@ -36,11 +36,11 @@ object FlagStat extends BDGCommandCompanion {
 
 class FlagStatArgs extends Args4jBase {
   @Argument(required = true, metaVar = "INPUT", usage = "The ADAM data to return stats for", index = 0)
-  val inputPath: String = null
+  var inputPath: String = null
   @Args4jOption(required = false, name = "-o", usage = "Optionally write the stats to this file.")
-  val outputPath: String = null
+  var outputPath: String = null
   @Args4jOption(required = false, name = "-stringency", usage = "Set the parsing stringency: SILENT, LENIENT, STRICT.")
-  val stringency: String = "SILENT"
+  var stringency: String = "SILENT"
 }
 
 class FlagStat(protected val args: FlagStatArgs) extends BDGSparkCommand[FlagStatArgs] {

adam-python/bdgenomics/adam/test/__init__.py

@@ -57,7 +57,10 @@ def checkFiles(self, file1, file2):
     def setUp(self):
         self._old_sys_path = list(sys.path)
         class_name = self.__class__.__name__
-        self.ss = SparkSession.builder.master('local[4]').appName(class_name).getOrCreate()
+        self.ss = SparkSession.builder \
+                              .master('local[4]') \
+                              .appName(class_name) \
+                              .getOrCreate()
         self.sc = self.ss.sparkContext
 
 

pom.xml

@@ -588,6 +588,11 @@
         <artifactId>adam-codegen-spark2_${scala.version.prefix}</artifactId>
         <version>${project.version}</version>
       </dependency>
+      <dependency>
+        <groupId>com.thoughtworks.paranamer</groupId>
+        <artifactId>paranamer</artifactId>
+        <version>2.8</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>