Merge pull request #3 from recipegrace/develop

error in runJAR with jars not in Bucket
recipegrace · Sep 11, 2018 · c2d4859 · c2d4859
2 parents 1d67b1c + f9555bc
commit c2d4859
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 7 deletions.
diff --git a/files/javajob.bbc b/files/javajob.bbc
@@ -8,7 +8,7 @@ zone="us-east1-c"
     javajob javajobtest {
         mainClass="com.recipegrace.biglibrary.gasexamples.SimpleGoogleCloudOps"
         args= "--input" ,"gs://hd-srch-dev-mosambi/terms.txt", "--output" ,"gs://hd-srch-dev-mosambi/work/test/awesome.txt"
-        jarLocation= "gs://hd-srch-dev-mosambi/GasExamples-0.0.7-SNAPSHOT.jar"
+        jarLocation= "gs://rc-srch-dev-mosambi/jars/Robo2-core-0.0.1-SNAPSHOT.jar"
     }
 
 

diff --git a/src/main/resources/templates/download-jar.ssp b/src/main/resources/templates/download-jar.ssp
@@ -6,11 +6,12 @@
 <%@ val location:String = evalObject.evaluateVariable(javaJob.javaJob.jarLocation,localVariables).toString%>
 <%@ val bucket:String = location.substring(5).split("/").head %>
 <%@ val objectName:String = location.substring(5).split("/").drop(1).mkString("/") %>
+<%@ val fileName:String = location.substring(5).split("/").last %>
 <%@ var name: String%>
 
   ${name} = gcs_download_operator.GoogleCloudStorageDownloadOperator(
    task_id='copy_jar_${name}',
    bucket='${bucket}',
    object='${objectName}',
-   filename='/tmp/${objectName}'
+   filename='/tmp/${fileName}'
   )
diff --git a/src/main/resources/templates/run-jar.ssp b/src/main/resources/templates/run-jar.ssp
@@ -8,8 +8,10 @@
 <%@ val bucket:String = location.substring(5).split("/").head %>
 <%@ val objectName:String = location.substring(5).split("/").drop(1).mkString("/") %>
 <%@ var name: String%>
+<%@ var command1:String = s"gsutil -m cp -r ${location} ."%>
+<%@ var command2:String = s"java -cp ${objectName} ${mainClass} ${ javaJob.javaJob.args.get.map(f=> evalObject.evaluateVariable(f,localVariables )).mkString(" ")}"%>
 
   ${name} = bash_operator.BashOperator(
    task_id='run_jar_${name}',
-   bash_command='java -cp /tmp/${objectName} ${mainClass} ${ javaJob.javaJob.args.get.map(f=> evalObject.evaluateVariable(f,localVariables )).mkString(" ")}'
+   bash_command='${command1};${command2}'
   )
diff --git a/src/main/scala/com/recipegrace/bbc/workflow/RunJavaFlowBlock.scala b/src/main/scala/com/recipegrace/bbc/workflow/RunJavaFlowBlock.scala
@@ -35,14 +35,16 @@ class RunJavaFlowBlock(javaJob: JavaJob, variables:Map[String,Expression], confi
     object evalObject extends ExpressionCreator
     val id = IDGenerator.autoId
     val defaultArgs = Map ( "programConfiguration"->configuration, "localVariables" -> variables,"evalObject" -> evalObject)
-    val copyJarKey = displayName +"_C"+id
+   /* val copyJarKey = displayName +"_C"+id
     val copyJarContent = Templates.translate("templates/download-jar.ssp",Map("name" -> copyJarKey,
       "javaJob" -> javaJob) ++defaultArgs)
-
+*/
     val runJarKey = displayName+"_R"+id
     val runJarContent = Templates.translate("templates/run-jar.ssp",Map("name" -> runJarKey,
       "javaJob" -> javaJob) ++defaultArgs)
 
-    List( KeyAndContent(copyJarKey,copyJarContent,true), KeyAndContent (runJarKey,runJarContent,true))
+    List(
+      //KeyAndContent(copyJarKey,copyJarContent,true),
+      KeyAndContent (runJarKey,runJarContent,true))
   }
 }
diff --git a/src/test/scala/com/recipegrace/bbc/composer/TemplateTest.scala b/src/test/scala/com/recipegrace/bbc/composer/TemplateTest.scala
@@ -37,6 +37,7 @@ class TemplateTest extends BaseBBCGrammarTest with ExpressionCreator{
 
     val job = parseAll(_javaJobBody, jobDefintiion).get
 
+    /*
     Templates.translate("templates/download-jar.ssp", Map("name" -> taskName, "localVariables" -> localVariables,
       "evalObject" -> evalObject,
       "programConfiguration" -> programConfiguration,
@@ -47,14 +48,15 @@ class TemplateTest extends BaseBBCGrammarTest with ExpressionCreator{
          |   object='${objectName}',
          |   filename='/tmp/${objectName}'
   )""".stripMargin
+  */
 
     Templates.translate("templates/run-jar.ssp", Map("name" -> taskName, "localVariables" -> localVariables,
       "evalObject" -> evalObject,
       "programConfiguration" -> programConfiguration,
       "javaJob" -> job._2)) shouldBe "\n" +
       s"""  $taskName = bash_operator.BashOperator(
          |   task_id='run_jar_${taskName}',
-         |   bash_command='java -cp /tmp/${objectName} $mainClass ${programArguments.mkString(" ")}'
+         |   bash_command='gsutil -m cp -r gs://${bucketName}/${objectName} .;java -cp ${objectName} $mainClass ${programArguments.mkString(" ")}'
   )""".stripMargin
   }
   test("delete gcs") {