Clone dxWDL runtime library to project (#174)

* Removing the dependency on the bc utility

src/main/resources/application.conf

@@ -1,3 +1,3 @@
 dxWDL {
-    version = "0.77"
+    version = "0.78"
 }

src/main/scala/dxWDL/Utils.scala

@@ -596,6 +596,48 @@ object Utils {
         throw new Exception(s"Failure to upload file ${path}")
     }
 
+    // copy asset to local project, if it isn't already here.
+    def cloneAsset(assetRecord: DXRecord,
+                   dxProject: DXProject,
+                   pkgName: String,
+                   rmtProject: DXProject,
+                   verbose: Verbose) : Unit = {
+        if (dxProject == rmtProject) {
+            trace(verbose.on, s"The asset ${pkgName} is from this project ${rmtProject.getId}, no need to clone")
+            return
+        }
+        trace(verbose.on, s"The asset ${pkgName} is from a different project ${rmtProject.getId}")
+
+        // clone
+        val req = JsObject( "objects" -> JsArray(JsString(assetRecord.getId)),
+                            "project" -> JsString(dxProject.getId),
+                            "destination" -> JsString("/"))
+        val rep = DXAPI.projectClone(rmtProject.getId,
+                                     jsonNodeOfJsValue(req),
+                                     classOf[JsonNode])
+        val repJs:JsValue = jsValueOfJsonNode(rep)
+
+        val exists = repJs.asJsObject.fields.get("exists") match {
+            case None => throw new Exception("API call did not returnd an exists field")
+            case Some(JsArray(x)) => x.map {
+                case JsString(id) => id
+                case _ => throw new Exception("bad type, not a string")
+            }.toVector
+            case other => throw new Exception(s"API call returned invalid exists field")
+        }
+        val existingRecords = exists.filter(_.startsWith("record-"))
+        existingRecords.size match {
+            case 0 =>
+                val localAssetRecord = DXRecord.getInstance(assetRecord.getId)
+                trace(verbose.on, s"Created ${localAssetRecord.getId} pointing to asset ${pkgName}")
+            case 1 =>
+                trace(verbose.on, s"The project already has a record pointing to asset ${pkgName}")
+            case _ =>
+                throw new Exception(s"clone returned too many existing records ${exists}")
+        }
+    }
+
+
     // Parse a dnanexus file descriptor. Examples:
     //
     // "$dnanexus_link": {

src/main/scala/dxWDL/compiler/Native.scala

@@ -444,41 +444,6 @@ case class Native(dxWDLrtId: String,
         }
     }
 
-    private def cloneAssetFromDifferentProject(assetRecord: DXRecord,
-                                               desc: DXDataObject.Describe,
-                                               pkgName: String,
-                                               rmtProject: DXProject) : Unit = {
-        trace(verbose.on, s"The asset ${pkgName} is from a different project ${rmtProject.getId}")
-
-        // clone
-        val req = JsObject( "objects" -> JsArray(JsString(assetRecord.getId)),
-                            "project" -> JsString(dxProject.getId),
-                            "destination" -> JsString("/"))
-        val rep = DXAPI.projectClone(rmtProject.getId,
-                                     jsonNodeOfJsValue(req),
-                                     classOf[JsonNode])
-        val repJs:JsValue = jsValueOfJsonNode(rep)
-
-        val exists = repJs.asJsObject.fields.get("exists") match {
-            case None => throw new Exception("API call did not returnd an exists field")
-            case Some(JsArray(x)) => x.map {
-                case JsString(id) => id
-                case _ => throw new Exception("bad type, not a string")
-            }.toVector
-            case other => throw new Exception(s"API call returned invalid exists field")
-        }
-        val existingRecords = exists.filter(_.startsWith("record-"))
-        existingRecords.size match {
-            case 0 =>
-                val localAssetRecord = DXRecord.getInstance(assetRecord.getId)
-                trace(verbose.on, s"Created ${localAssetRecord.getId} pointing to asset ${pkgName}")
-            case 1 =>
-                trace(verbose.on, s"The project already has a record pointing to asset ${pkgName}")
-            case _ =>
-                throw new Exception(s"clone returned too many existing records ${exists}")
-        }
-    }
-
     // Set the run spec.
     //
     private def calcRunSpec(bashScript: String,
@@ -536,8 +501,7 @@ case class Native(dxWDLrtId: String,
                 if (!rmtContainer.isInstanceOf[DXProject])
                     throw new Exception(s"remote asset is in container ${rmtContainer.getId}, not a project")
                 val rmtProject = rmtContainer.asInstanceOf[DXProject]
-                if (rmtProject != dxProject)
-                    cloneAssetFromDifferentProject(dxRecord, desc, pkgName, rmtProject)
+                Utils.cloneAsset(dxRecord, dxProject, pkgName, rmtProject, verbose)
                 Some(JsObject("name" -> JsString(pkgName),
                               "id" -> jsValueOfJsonNode(pkgFile.getLinkAsJson)))
         }

src/main/scala/dxWDL/compiler/Top.scala

@@ -3,7 +3,7 @@
 
 package dxWDL.compiler
 
-import com.dnanexus.{DXDataObject, DXProject, DXSearch}
+import com.dnanexus.{DXDataObject, DXProject, DXRecord, DXSearch}
 import dxWDL.{CompilerOptions, CompilationResults, DxPath, InstanceTypeDB, Utils, Verbose}
 import dxWDL.Utils.DX_WDL_ASSET
 import java.nio.file.{Files, Path, Paths}
@@ -226,6 +226,22 @@ object Top {
         found(0).getId
     }
 
+    // We need the dxWDL runtime library cloned into this project, so it will
+    // be available to all subjobs we run.
+    private def cloneRtLibraryToProject(region: String,
+                                        dxWDLrtId: String,
+                                        dxProject: DXProject,
+                                        verbose: Verbose) : Unit = {
+        val region2project = Utils.getRegions()
+        val (projNameRt, folder)  = getProjectWithRuntimeLibrary(region2project, region)
+        val dxProjRt = DxPath.lookupProject(projNameRt)
+        Utils.cloneAsset(DXRecord.getInstance(dxWDLrtId),
+                         dxProject,
+                         DX_WDL_ASSET,
+                         dxProjRt,
+                         verbose)
+    }
+
     // Backend compiler pass
     private def compileNative(irNs: IR.Namespace,
                               folder: String,
@@ -234,6 +250,7 @@ object Top {
         // get billTo and region from the project
         val (billTo, region) = Utils.projectDescribeExtraInfo(dxProject)
         val dxWDLrtId = getAssetId(region, cOpt.verbose)
+        cloneRtLibraryToProject(region, dxWDLrtId, dxProject, cOpt.verbose)
 
         // get list of available instance types
         val instanceTypeDB = InstanceTypeDB.query(dxProject, cOpt.verbose)

test/basic/instance_types.wdl

@@ -5,7 +5,7 @@ task DiskSpaceSpec {
   command <<<
     lines=$(df -t btrfs | grep dev)
     size_kb=$(echo $lines | cut -d ' ' -f 2)
-    size_gb=$(echo "$size_kb / (1024 * 1024)" | bc)
+    let "size_gb= $size_kb / (1024 * 1024)"
     if [[ $size_gb -ge disk_req_gb ]]; then
        echo "true"
     else
@@ -31,7 +31,7 @@ task DiskSpaceTaskDeclarations {
   command <<<
     lines=$(df -t btrfs | grep dev)
     size_kb=$(echo $lines | cut -d ' ' -f 2)
-    size_gb=$(echo "$size_kb / (1024 * 1024)" | bc)
+    let "size_gb= $size_kb / (1024 * 1024)"
     if [[ $size_gb -ge disk_req_gb ]]; then
        echo "true"
     else
@@ -54,7 +54,7 @@ task MemorySpec {
   command <<<
     line=$(cat /proc/meminfo | grep MemTotal)
     size_kb=$(echo $line | cut -d ' ' -f 2)
-    size_gb=$(echo "$size_kb / (1024 * 1024)" | bc)
+    let "size_gb= $size_kb / (1024 * 1024)"
     if [[ $size_gb -ge $memory_req_gb ]]; then
        echo "true"
     else
@@ -130,7 +130,7 @@ task Shortcut {
     command {
         line=$(cat /proc/meminfo | grep MemTotal)
         size_kb=$(echo $line | cut -d ' ' -f 2)
-        size_gb=$(echo "$size_kb / (1024 * 1024)" | bc)
+        let "size_gb= $size_kb / (1024 * 1024)"
         echo $size_gb
     }
     runtime {

test/docker/platform_asset.wdl

@@ -10,7 +10,7 @@ task image {
         echo "Major Major"
     >>>
     runtime {
-        docker: "dx://dxWDL_playground:/ubuntu"
+        docker: "dx://dxWDL_playground:/test_data/ubuntu"
     }
     output {
         String result = read_string(stdout())