Merge branch 'dev'

Conflicts:
	README.md

.gitignore

@@ -0,0 +1 @@
+tags

.travis.yml

@@ -0,0 +1,2 @@
+language: java
+script:  ant test

README.md

@@ -1,4 +1,268 @@
-wdl
-===
+Workflow Description Language (wdl)
+===================================
 
-Workflow Description Language
+The Workflow Description Language is a language for describing dependency trees of tasks (algorithms) in a concise and clear syntax.
+
+Installation
+============
+
+Generating the JAR
+------------------
+
+To build the JAR file, run:
+
+```
+$ mvn package
+```
+
+Which will create a file target/Wdl-${version}.jar as an executable JAR.  To invoke the CLI:
+
+```
+$ java -jar target/Wdl-0.0.1.jar examples/0.wdl ast
+```
+
+Generating the parser code
+--------------------------
+
+Use the Hermes parser generator (http://github.com/scottfrazer/hermes).  From the root of this repository, run:
+
+```
+$ hermes generate grammars/composite_task.zgr --directory=src/main/java/org/broadinstitute/compositetask --language=java --java-package=org.broadinstitute.compositetask
+```
+
+Or, run the Ant target `generate-parser` which will run the command above:
+
+```
+$ ant generate-parser
+```
+
+Usage
+=====
+
+Java API
+--------
+
+From Java code, the main interface is the CompositeTask, which can be used in this example to print out the immediate children nodes of this composite task:
+
+```java
+CompositeTask ct = new CompositeTask(new File(args[0]));
+for ( CompositeTaskNode entry : ct.getNodes() ) {
+  System.out.println("Node: " + entry);
+}
+```
+
+Command-line Interface
+----------------------
+
+The data file we'll use is:
+
+```
+$ cat examples/7.wdl
+composite_task test {
+  step s0[version=0] {
+    output: File("abc") as foo;
+  }
+
+  for (I in L) {
+    for (J in M) {
+      step s1[version=0] {
+        input: p0=I, p1=J, p2=foo;
+        output: File("def") as bar;
+      }
+    }
+  }
+
+  step s2[version=0] {
+    input: p0=bar;
+  }
+}
+```
+
+Get the abstract syntax tree:
+
+```
+$ java -jar dist/Wdl-0.0.1.jar examples/7.wdl ast
+(CompositeTask:
+  body=[
+    (Step:
+      body=[
+        (StepOutputList:
+          outputs=[
+            (StepFileOutput:
+              as=(Variable:
+                member=None,
+                name=identifier
+              ),
+              file=string
+            )
+          ]
+        )
+      ],
+      task=(Task:
+        attributes=[
+          (TaskAttribute:
+            value=number,
+            key=identifier
+          )
+        ],
+        name=identifier
+      ),
+      name=None
+    ),
+    (ForLoop:
+      body=[
+        (ForLoop:
+          body=[
+            (Step:
+              body=[
+                (StepInputList:
+                  inputs=[
+                    (StepInput:
+                      parameter=identifier,
+                      value=(Variable:
+                        member=None,
+                        name=identifier
+                      )
+                    ),
+                    (StepInput:
+                      parameter=identifier,
+                      value=(Variable:
+                        member=None,
+                        name=identifier
+                      )
+                    ),
+                    (StepInput:
+                      parameter=identifier,
+                      value=(Variable:
+                        member=None,
+                        name=identifier
+                      )
+                    )
+                  ]
+                ),
+                (StepOutputList:
+                  outputs=[
+                    (StepFileOutput:
+                      as=(Variable:
+                        member=None,
+                        name=identifier
+                      ),
+                      file=string
+                    )
+                  ]
+                )
+              ],
+              task=(Task:
+                attributes=[
+                  (TaskAttribute:
+                    value=number,
+                    key=identifier
+                  )
+                ],
+                name=identifier
+              ),
+              name=None
+            )
+          ],
+          item=identifier,
+          collection=identifier
+        )
+      ],
+      item=identifier,
+      collection=identifier
+    ),
+    (Step:
+      body=[
+        (StepInputList:
+          inputs=[
+            (StepInput:
+              parameter=identifier,
+              value=(Variable:
+                member=None,
+                name=identifier
+              )
+            )
+          ]
+        )
+      ],
+      task=(Task:
+        attributes=[
+          (TaskAttribute:
+            value=number,
+            key=identifier
+          )
+        ],
+        name=identifier
+      ),
+      name=None
+    )
+  ],
+  name=identifier
+)
+```
+
+Get a view of the graph
+
+```
+$ java -jar dist/Wdl-0.0.1.jar examples/7.wdl graph
+VERTICIES
+---------
+[Step: name=s1]
+[Variable: name=J]
+[Variable: name=M]
+[Variable: name=I]
+[Variable: name=L]
+[Variable: name=foo]
+[Step: name=s0]
+[Variable: name=bar]
+[Step: name=s2]
+[CompositeTaskForScope: collection=[Variable: name=L], var=[Variable: name=I], # nodes=1]
+[CompositeTaskForScope: collection=[Variable: name=M], var=[Variable: name=J], # nodes=1]
+
+EDGES
+-----
+[Edge
+  from: [Step: name=s1]
+  to: [Variable: name=bar]
+]
+[Edge
+  from: [Variable: name=L]
+  to: [CompositeTaskForScope: collection=[Variable: name=L], var=[Variable: name=I], # nodes=1]
+]
+[Edge
+  from: [Variable: name=bar]
+  to: [Step: name=s2]
+]
+[Edge
+  from: [Variable: name=foo]
+  to: [Step: name=s1]
+]
+[Edge
+  from: [Step: name=s0]
+  to: [Variable: name=foo]
+]
+[Edge
+  from: [CompositeTaskForScope: collection=[Variable: name=M], var=[Variable: name=J], # nodes=1]
+  to: [Variable: name=J]
+]
+[Edge
+  from: [Variable: name=J]
+  to: [Step: name=s1]
+]
+[Edge
+  from: [Variable: name=M]
+  to: [CompositeTaskForScope: collection=[Variable: name=M], var=[Variable: name=J], # nodes=1]
+]
+[Edge
+  from: [CompositeTaskForScope: collection=[Variable: name=L], var=[Variable: name=I], # nodes=1]
+  to: [Step: name=s2]
+]
+[Edge
+  from: [Variable: name=I]
+  to: [Step: name=s1]
+]
+[Edge
+  from: [CompositeTaskForScope: collection=[Variable: name=L], var=[Variable: name=I], # nodes=1]
+  to: [Variable: name=I]
+]
+```

build.xml

@@ -0,0 +1,20 @@
+<project name="wdl">
+  <target name="generate-parser" description="Generate the parser from grammar file">
+        <property name="parser.dir" value="src/main/java/org/broadinstitute/parser" />
+        <property name="parser.package" value="org.broadinstitute.parser" />
+        <property name="parser.grammar" value="grammars/composite_task.zgr" />
+
+        <exec executable="hermes">
+            <arg value="--version" />
+        </exec>
+        <echo message="Generating parser in ${parser.dir}" />
+        <mkdir dir="${parser.dir}" />
+        <exec executable="hermes">
+            <arg value="generate" />
+            <arg value="${parser.grammar}" />
+            <arg value="--directory=${parser.dir}" />
+            <arg value="--language=java" />
+            <arg value="--java-package=${parser.package}" />
+        </exec>
+    </target>
+</project>

examples/CopyNumberQC.wdl

@@ -0,0 +1,18 @@
+composite_task CopyNumberQC {
+  step LaneBlackList[version=6] {
+    output: File("lane_blacklist.txt") as lane_blacklist;
+  }
+  for ( sample in samples ) {
+    step RegionCovPerLane[version=16] {
+      input: bam_file=sample.bam, sample_id=sample.id;
+      output: File("${sample.id}.rcl") into rcl;
+    }
+    step MakeLaneList[version=11] {
+      input: bam_file=sample.bam, sample_id=sample.id;
+      output: File("${sample.id}.lanelist") into lanelist;
+    }
+  }
+  step CopyNumberQC[version=25] {
+    input: lanes_list=lanelist, rcl_list=rcl, lane_blacklist=lane_blacklist;
+  }
+}

examples/MutSig.wdl

@@ -0,0 +1,27 @@
+/* inputs:
+ *   1) MutSigPreprocess.{individual_set_id, maf1, maf2, maf3, maf4, maflabel1, maflabel2, maflabel3, maflabel4, wig1, wig2, wig3, wig4, build, context65_dir, build_dir, num_categories, target_list, paramfile}
+ *   2) ProcessCoverageForMutSig has no variables.  All inputs come from prior step.
+ *   3) MutSigRun.{individual_set_id, gene_list, build, geneset_file, cosmic_file, refseq_file, build_dir, param_file, jobcount}
+ */
+
+composite_task MutSig {
+
+  step MutSigPreprocess[version=86] {
+    output: File("coverage.prepare.txt") as coverage_prepare_file,
+            File("patients.txt") as patient_list,
+            File("${individual_set_id}.maf") as mutation_list,
+            File("mutation_preprocessing_report.txt") as mutation_preprocessing_report;
+  }
+
+  step ProcessCoverageForMutSig[version=75] {
+    input: coverage=coverage_prepare_file, patients=patient_list, mutations=mutation_list;
+    output: File("coverage.mat") as coverage_file,
+            File("mutcategs.txt") as category_file;
+  }
+
+  step MutSigRun[version=157] as MutSig {
+    input: mutation_list=mutation_list, coverage_file=coverage_file, patients=patient_list, category_file=category_file, mutation_preprocessing_report=foobar;
+    output: File("bargraphs.png") as graphs;
+  }
+
+}

grammars/composite_task.zgr

@@ -0,0 +1,28 @@
+{
+  "ll1": {
+    "start": "wdl",
+    "rules": [
+      "wdl := list(wdl_entity)",
+      "wdl_entity := composite_task",
+      "composite_task := 'composite_task' + 'identifier' + 'lbrace' + list(composite_task_entity) + 'rbrace' -> CompositeTask( name=$1, body=$3 )",
+      "composite_task_entity := step | for_loop | composite_task",
+      "for_loop := 'for' + 'lparen' + 'identifier' + 'in' + 'identifier' + 'rparen' + 'lbrace' + list(composite_task_entity) + 'rbrace' -> ForLoop( collection=$4, item=$2, body=$7 )",
+      "step := 'step' + task_identifier + optional(step_name) + 'lbrace' + list(step_attr) + 'rbrace' -> Step( task=$1, name=$2, body=$4 )",
+      "task_identifier := 'identifier' + optional(task_attrs) -> Task(name=$0, attributes=$1)",
+      "task_attrs := 'lsquare' + list(task_attr) + 'rsquare' -> $1",
+      "task_attr := 'identifier' + 'assign' + task_attr_value -> TaskAttribute(key=$0, value=$2)",
+      "task_attr_value := 'identifier' | 'string' | 'number'",
+      "step_name := 'as' + 'identifier' -> $1",
+      "step_attr := step_input_list | step_output_list",
+      "step_input_list := 'input' + 'colon' + list(step_input, 'comma')  + 'semi' -> StepInputList( inputs=$2 )",
+      "step_input := 'identifier' + 'assign' + variable -> StepInput(parameter=$0, value=$2)",
+      "step_output_list := 'output' + 'colon' + list(step_output, 'comma') + 'semi' -> StepOutputList( outputs=$2 )",
+      "step_output := step_output_mode + 'lparen' + 'string' + 'rparen' + step_output_location -> StepOutput( mode=$0, expression=$2, var=$4 )",
+      "step_output_mode := 'file' | 'first_line'",
+      "step_output_location :=  'as' + variable -> OutputVariable( var=$1 )",
+      "step_output_location :=  'into' + variable -> OutputListAppend( var=$1 )",
+      "variable := 'identifier' + optional(variable_member) -> Variable(name=$0, member=$1)",
+      "variable_member := 'dot' + 'identifier -> $1"
+    ]
+  }
+}