Merge pull request #319 from SeqWare/feature/efficiency_experiment2

Feature/efficiency experiment2
SeqWare · Jul 16, 2015 · 4e49161 · 4e49161
2 parents 8b57b7f + fdcb7ef
commit 4e49161
Show file tree

Hide file tree

Showing 10 changed files with 39 additions and 37 deletions.
diff --git a/...kflow/src/main/resources/archetype-resources/src/main/java/__workflow-name__Workflow.java b/...kflow/src/main/resources/archetype-resources/src/main/java/__workflow-name__Workflow.java
@@ -74,15 +74,15 @@ public void buildWorkflow() {
 
         // a simple bash job to call mkdir
 	// note that this job uses the system's mkdir (which depends on the system being *nix)
-	// this also translates into a 4000 h_vmem limit when using sge 
-        Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("4000");
+	// this also translates into a 3000 h_vmem limit when using sge 
+        Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("3000");
         mkdirJob.getCommand().addArgument("mkdir test1");      
 
 	String inputFilePath = this.getFiles().get("file_in_0").getProvisionedPath();
 
         // a simple bash job to cat a file into a test file
 	// the file is not saved to the metadata database
-        Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+        Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
         copyJob1.setCommand(catPath + " " + inputFilePath + "> test1/test.out");
         copyJob1.addParent(mkdirJob);
 	// this will annotate the processing event associated with the cat of the file above
@@ -91,7 +91,7 @@ public void buildWorkflow() {
 
         // a simple bash job to echo to an output file and concat an input file
 	// the file IS saved to the metadata database
-        Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+        Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
 	copyJob2.getCommand().addArgument(echoPath).addArgument(greeting).addArgument(" > ").addArgument("dir1/output");
 	copyJob2.getCommand().addArgument(";");
 	copyJob2.getCommand().addArgument(catPath + " " +inputFilePath+ " >> dir1/output");

diff --git a/seqware-common/src/main/java/io/seqware/pipeline/SqwKeys.java b/seqware-common/src/main/java/io/seqware/pipeline/SqwKeys.java
@@ -67,7 +67,7 @@ public enum SqwKeys {
                     + "If present, ${threads} will be replaced with the job-specific value.", "-pe serial ${threads}"),
     OOZIE_SGE_MAX_MEMORY_PARAM_FORMAT(null , Categories.LAUNCH, true, "Format of qsub flag for specifying the max memory. "
             + "If present, ${maxMemory} will be replaced with the job-specific value.", "-l h_vmem=${maxMemory}M"),
-    SW_CONTROL_NODE_MEMORY(null, Categories.ADMIN, false, "In atypical environments, the default h_vmem constraint for SGE is too stringent. Override them using this (units in megabytes)", "4000"),
+    SW_CONTROL_NODE_MEMORY(null, Categories.ADMIN, false, "In atypical environments, the default h_vmem constraint for SGE is too stringent. Override them using this (units in megabytes)", "3000"),
     SW_ADMIN_REST_URL(null, Categories.ADMIN, false, "Location of the admin web service, currently used for deletion", "http://localhost:38080/seqware-admin-webservice"),
     SW_LOCK_ID(null, Categories.ADMIN, false, "Used to override the JUnique lock used to ensure that utilities don't run concurrently","seqware"),
     SW_ENCRYPT_KEY(null, Categories.ADMIN, false,"Legacy key used to encrypt provisioned files", "seqware"),

diff --git a/...are-distribution/docs/site/content/docs/3-getting-started/developer-tutorial.md b/...are-distribution/docs/site/content/docs/3-getting-started/developer-tutorial.md
@@ -249,20 +249,20 @@ dateJob is now the final job that manipulates the dir1/output file.
 
         // a simple bash job to call mkdir
         // note that this job uses the system's mkdir (which depends on the system being *nix)
-        Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("4000");
+        Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("3000");
         mkdirJob.getCommand().addArgument("mkdir test1");
 
         String inputFilePath = this.getFiles().get("file_in_0").getProvisionedPath();
 
         // a simple bash job to cat a file into a test file
         // the file is not saved to the metadata database
-        Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+        Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
         copyJob1.setCommand(catPath + " " + inputFilePath + "> test1/test.out");
         copyJob1.addParent(mkdirJob);
 
         // a simple bash job to echo to an output file and concat an input file
         // the file IS saved to the metadata database
-        Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+        Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
         copyJob2.getCommand().addArgument(echoPath).addArgument(greeting).addArgument(" > ").addArgument("dir1/output");
         copyJob2.getCommand().addArgument(";");
         copyJob2.getCommand().addArgument(catPath + " " +inputFilePath+ " >> dir1/output");
@@ -272,7 +272,7 @@ dateJob is now the final job that manipulates the dir1/output file.
         outputFile.getAnnotations().put("provision.file.annotation.key.1", "provision.annotation.value.1");
 
 
-	Job dateJob = this.getWorkflow().createBashJob("date").setMaxMemory("4000");
+	Job dateJob = this.getWorkflow().createBashJob("date").setMaxMemory("3000");
 	dateJob.setCommand("date >> dir1/output");
 	dateJob.addParent(copyJob2);
         dateJob.addFile(outputFile);

diff --git a/seqware-distribution/docs/site/content/docs/6-pipeline/debug-workflows.md b/seqware-distribution/docs/site/content/docs/6-pipeline/debug-workflows.md
@@ -126,21 +126,21 @@ Next, modify the Java workflow class to have a large number of errors
 
 		// a simple bash job to call mkdir
 		// note that this job uses the system's mkdir (which depends on the system being *nix)
-		// this also translates into a 4000 h_vmem limit when using sge
-		Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("4000");
+		// this also translates into a 3000 h_vmem limit when using sge
+		Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("3000");
 		mkdirJob.getCommand().addArgument("mkdir test1");
 
 		String inputFilePath = this.getFiles().get("file_in_0").getProvisionedPath();
 
 		// a simple bash job to cat a file into a test file
 		// the file is not saved to the metadata database
-		Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+		Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
 		copyJob1.setCommand(catPath + " " + inputFilePath + "> test1/test.out");
 		copyJob1.addParent(mkdirJob);
 
 		// a simple bash job to echo to an output file and concat an input file
 		// the file IS saved to the metadata database
-		Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+		Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
 		copyJob2.getCommand().addArgument(echoPath).addArgument(greeting).addArgument(" > ").addArgument("dir1/output");
 		copyJob2.getCommand().addArgument(";");
 		copyJob2.getCommand().addArgument(catPath + " " + inputFilePath + " >> dir1/output");
@@ -279,23 +279,23 @@ The previous section concerned workflows that fail during launch time when the w
 
 		// a simple bash job to call mkdir
 		// note that this job uses the system's mkdir (which depends on the system being *nix)
-		// this also translates into a 4000 h_vmem limit when using sge
-		Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("4000");
+		// this also translates into a 3000 h_vmem limit when using sge
+		Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("3000");
 		mkdirJob.getCommand().addArgument("mkdir $fubar_test");
 		// mkdirJob.getCommand().addArgument("mkdir test1");
 
 		String inputFilePath = this.getFiles().get("file_in_0").getProvisionedPath();
 
 		// a simple bash job to cat a file into a test file
 		// the file is not saved to the metadata database
-		Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+		Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
 		copyJob1.setCommand(catPath + " " + inputFilePath + "> fubar/test.out");
 		// copyJob1.setCommand(catPath + " " + inputFilePath + "> test1/test.out");
 		copyJob1.addParent(mkdirJob);
 
 		// a simple bash job to echo to an output file and concat an input file
 		// the file IS saved to the metadata database
-		Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+		Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
 		copyJob2.getCommand().addArgument(echoPath).addArgument(greeting).addArgument(" > ").addArgument("fubar/output");
 		// copyJob2.getCommand().addArgument(echoPath).addArgument(greeting).addArgument(" > ").addArgument("dir1/output");
 		copyJob2.getCommand().addArgument(";");

diff --git a/...re-distribution/docs/site/layouts/includes/java_workflows/java_workflow_jobs.md b/...re-distribution/docs/site/layouts/includes/java_workflows/java_workflow_jobs.md
@@ -4,15 +4,15 @@
 
         // a simple bash job to call mkdir
 	// note that this job uses the system's mkdir (which depends on the system being *nix)
-	// this also translates into a 4000 h_vmem limit when using sge 
-        Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("4000");
+	// this also translates into a 3000 h_vmem limit when using sge 
+        Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("3000");
         mkdirJob.getCommand().addArgument("mkdir test1");      
 
 	String inputFilePath = this.getFiles().get("file_in_0").getProvisionedPath();
 
         // a simple bash job to cat a file into a test file
 	// the file is not saved to the metadata database
-        Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+        Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
         copyJob1.setCommand(catPath + " " + inputFilePath + "> test1/test.out");
         copyJob1.addParent(mkdirJob);
 	// this will annotate the processing event associated with the cat of the file above
@@ -21,7 +21,7 @@
 
         // a simple bash job to echo to an output file and concat an input file
 	// the file IS saved to the metadata database
-        Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+        Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
 	copyJob2.getCommand().addArgument(echoPath).addArgument(greeting).addArgument(" > ").addArgument("dir1/output");
 	copyJob2.getCommand().addArgument(";");
 	copyJob2.getCommand().addArgument(catPath + " " +inputFilePath+ " >> dir1/output");

diff --git a/...rc/test/resources/net/sourceforge/seqware/pipeline/debugging_tutorial/FailLaunch.template b/...rc/test/resources/net/sourceforge/seqware/pipeline/debugging_tutorial/FailLaunch.template
@@ -79,21 +79,21 @@ public class BuggyWorkflowWorkflow extends AbstractWorkflowDataModel {
 
 	// a simple bash job to call mkdir
 	// note that this job uses the system's mkdir (which depends on the system being *nix)
-	// this also translates into a 4000 h_vmem limit when using sge
-	Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("4000");
+	// this also translates into a 3000 h_vmem limit when using sge
+	Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("3000");
 	mkdirJob.getCommand().addArgument("mkdir test1");
 
 	String inputFilePath = this.getFiles().get("file_in_0").getProvisionedPath();
 
 	// a simple bash job to cat a file into a test file
 	// the file is not saved to the metadata database
-	Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+	Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
 	copyJob1.setCommand(catPath + " " + inputFilePath + "> test1/test.out");
 	copyJob1.addParent(mkdirJob);
 
 	// a simple bash job to echo to an output file and concat an input file
 	// the file IS saved to the metadata database
-	Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+	Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
 	copyJob2.getCommand().addArgument(echoPath).addArgument(greeting).addArgument(" > ").addArgument("dir1/output");
 	copyJob2.getCommand().addArgument(";");
 	copyJob2.getCommand().addArgument(catPath + " " + inputFilePath + " >> dir1/output");
@@ -123,4 +123,4 @@ public class BuggyWorkflowWorkflow extends AbstractWorkflowDataModel {
 	return file1;
     }
 
-}
+}
diff --git a/...g/src/test/resources/net/sourceforge/seqware/pipeline/debugging_tutorial/FailRun.template b/...g/src/test/resources/net/sourceforge/seqware/pipeline/debugging_tutorial/FailRun.template
@@ -61,23 +61,23 @@ public class BuggyWorkflowWorkflow extends AbstractWorkflowDataModel {
 
 	// a simple bash job to call mkdir
 	// note that this job uses the system's mkdir (which depends on the system being *nix)
-	// this also translates into a 4000 h_vmem limit when using sge
-	Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("4000");
+	// this also translates into a 3000 h_vmem limit when using sge
+	Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("3000");
 	mkdirJob.getCommand().addArgument("mkdir $fubar_test");
 	// mkdirJob.getCommand().addArgument("mkdir test1");
 
 	String inputFilePath = this.getFiles().get("file_in_0").getProvisionedPath();
 
 	// a simple bash job to cat a file into a test file
 	// the file is not saved to the metadata database
-	Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+	Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
 	copyJob1.setCommand(catPath + " " + inputFilePath + "> fubar/test.out");
 	// copyJob1.setCommand(catPath + " " + inputFilePath + "> test1/test.out");
 	copyJob1.addParent(mkdirJob);
 
 	// a simple bash job to echo to an output file and concat an input file
 	// the file IS saved to the metadata database
-	Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+	Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
 	copyJob2.getCommand().addArgument(echoPath).addArgument(greeting).addArgument(" > ").addArgument("fubar/output");
 	// copyJob2.getCommand().addArgument(echoPath).addArgument(greeting).addArgument(" > ").addArgument("dir1/output");
 	copyJob2.getCommand().addArgument(";");
@@ -104,4 +104,4 @@ public class BuggyWorkflowWorkflow extends AbstractWorkflowDataModel {
 	return file1;
     }
 
-}
+}
diff --git a/...c/test/resources/net/sourceforge/seqware/pipeline/regression_testing/seqware1890.template b/...c/test/resources/net/sourceforge/seqware/pipeline/regression_testing/seqware1890.template
@@ -60,26 +60,26 @@ public class HelloOutofOrderWorkflow extends AbstractWorkflowDataModel {
 
         // a simple bash job to call mkdir
         // note that this job uses the system's mkdir (which depends on the system being *nix)
-        // this also translates into a 4000 h_vmem limit when using sge
-        Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("4000");
+        // this also translates into a 3000 h_vmem limit when using sge
+        Job mkdirJob = this.getWorkflow().createBashJob("bash_mkdir").setMaxMemory("3000");
         mkdirJob.getCommand().addArgument("mkdir test1");
 
         String inputFilePath = this.getFiles().get("file_in_0").getProvisionedPath();
 
         // a simple bash job to cat a file into a test file
         // the file is not saved to the metadata database
-        Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+        Job copyJob1 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
         copyJob1.setCommand(catPath + " " + inputFilePath + "> test1/test.out");
         copyJob1.addParent(mkdirJob);
 
         // create this out of order
-        Job mkdirJob2 = this.getWorkflow().createBashJob("bash_mkdir2").setMaxMemory("4000");
+        Job mkdirJob2 = this.getWorkflow().createBashJob("bash_mkdir2").setMaxMemory("3000");
         mkdirJob2.getCommand().addArgument("mkdir funky_test");
         copyJob1.addParent(mkdirJob2);
 
         // a simple bash job to echo to an output file and concat an input file
         // the file IS saved to the metadata database
-        Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("4000");
+        Job copyJob2 = this.getWorkflow().createBashJob("bash_cp").setMaxMemory("3000");
         copyJob2.getCommand().addArgument(echoPath).addArgument(greeting).addArgument(" > ").addArgument("dir1/output");
         copyJob2.getCommand().addArgument(";");
         copyJob2.getCommand().addArgument(catPath + " " + inputFilePath + " >> dir1/output");

diff --git a/...in/java/net/sourceforge/seqware/pipeline/workflowV2/engine/oozie/object/OozieBashJob.java b/...in/java/net/sourceforge/seqware/pipeline/workflowV2/engine/oozie/object/OozieBashJob.java
@@ -118,6 +118,7 @@ public ArrayList<String> generateRunnerLine() {
         ArrayList<String> args = new ArrayList<>();
         String pathToJRE = createPathToJava();
         args.add(pathToJRE + "java");
+        args.add("-XX:+UseSerialGC");
         args.add("-Xmx" + jobObj.getCommand().getMaxMemory());
         args.add("-classpath");
         args.add(seqwareJarPath);

diff --git a/...et/sourceforge/seqware/pipeline/workflowV2/engine/oozie/object/OozieProvisionFileJob.java b/...et/sourceforge/seqware/pipeline/workflowV2/engine/oozie/object/OozieProvisionFileJob.java
@@ -73,7 +73,7 @@ private List<String> runnerArgs() {
          * So, despite the fact that ProvisionFiles knows the destination of the file, we still need the following since ProvisionFiles
          * reports just the filename as the destination, and then Runner prepends that file longName with the value of the following.
          * Madness.
-         * 
+         *
          * Based on code from pegasus.object.ProvisionFilesJob.buildCommandString()
          */
         if (file.getOutputPath() == null) {
@@ -149,6 +149,7 @@ public ArrayList<String> generateRunnerLine() {
         ArrayList<String> args = new ArrayList<>();
         String pathToJRE = createPathToJava();
         args.add(pathToJRE + "java");
+        args.add("-XX:+UseSerialGC");
         args.add("-Xmx" + jobObj.getCommand().getMaxMemory());
         args.add("-classpath");
         args.add(seqwareJarPath);