Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

We’re showing branches in this repository, but you can also compare across forks.

base: d7e1c1f998
...
compare: 3106322a72
  • 8 commits
  • 31 files changed
  • 0 commit comments
  • 1 contributor
Showing with 248 additions and 452 deletions.
  1. +30 −14 src/main/scala/ducttape.scala
  2. +2 −2 src/main/scala/ducttape/exec/BuildEnvironment.scala
  3. +1 −3 src/main/scala/ducttape/exec/CompletionChecker.scala
  4. +9 −2 src/main/scala/ducttape/exec/DirectoryArchitect.scala
  5. +5 −5 src/main/scala/ducttape/exec/Executor.scala
  6. +14 −11 src/main/scala/ducttape/exec/PackageBuilder.scala
  7. +2 −4 src/main/scala/ducttape/exec/PartialOutputRemover.scala
  8. +28 −12 src/main/scala/ducttape/exec/TaskEnvironment.scala
  9. +55 −12 src/main/scala/ducttape/syntax/AbstractSyntaxTree.scala
  10. +3 −0  src/main/scala/ducttape/syntax/FileFormatException.scala
  11. +5 −7 src/main/scala/ducttape/syntax/Grammar.scala
  12. +13 −2 src/main/scala/ducttape/syntax/GrammarParser.scala
  13. +3 −2 src/main/scala/ducttape/syntax/bash.scala
  14. +0 −20 src/main/scala/ducttape/workflow/hacks/Gimme.scala
  15. +1 −6 syntax/tutorial/2-packages/1-disk.tape
  16. +0 −31 syntax/tutorial/2-packages/1-disk.tape.TODO
  17. +11 −14 syntax/tutorial/2-packages/2-git.tape
  18. +0 −43 syntax/tutorial/2-packages/2-git.tape.TODO
  19. +0 −3  syntax/tutorial/2-packages/3-svn.tape
  20. +0 −44 syntax/tutorial/2-packages/3-svn.tape.TODO
  21. +0 −20 syntax/tutorial/2-packages/gimme/cdec
  22. +0 −24 syntax/tutorial/2-packages/gimme/gimme_repo_git
  23. +0 −21 syntax/tutorial/2-packages/gimme/gimme_repo_svn
  24. +0 −21 syntax/tutorial/2-packages/gimme/gimme_repo_symlink
  25. +0 −18 syntax/tutorial/2-packages/gimme/multeval
  26. +0 −17 syntax/tutorial/2-packages/gimme/sa_extract
  27. +0 −21 syntax/tutorial/2-packages/gimme/srilm
  28. +5 −0 syntax/tutorial/6-schedulers/1-shell.tape
  29. +48 −8 syntax/tutorial/6-schedulers/2-sge.tape
  30. +0 −65 syntax/tutorial/6-schedulers/2-sge.tape.TODO
  31. +13 −0 syntax/tutorial/6-schedulers/3-meta-config.XXX
44 src/main/scala/ducttape.scala
View
@@ -11,6 +11,7 @@ import ducttape.exec.PackageFinder
import ducttape.exec.TaskEnvironment
import ducttape.exec.UnpackedDagVisitor
import ducttape.exec.DirectoryArchitect
+import ducttape.exec.PackageVersioner
import ducttape.syntax.AbstractSyntaxTree._
import ducttape.syntax.GrammarParser
import ducttape.syntax.StaticChecker
@@ -28,6 +29,7 @@ import ducttape.workflow.Types._
import ducttape.util.Files
import ducttape.util.OrderedSet
import ducttape.util.MutableOrderedSet
+import ducttape.exec.FullTaskEnvironment
package ducttape {
class Config {
@@ -192,13 +194,11 @@ object Ducttape {
sys.exit(1)
throw new Error("Unreachable") // make the compiler happy
}
-/*
case e: Exception => {
err.println("%sERROR: %s".format(conf.errorColor, e.getMessage))
exit(1)
throw new Error("Unreachable") // make the compiler happy
}
-*/
case t: Throwable => throw t
}
}
@@ -359,10 +359,21 @@ object Ducttape {
err.println("Planned: " + plannedVertices)
// TODO: Refactor a bit? Only return the proper versioner? Make into on-demand method?
val (cc: CompletionChecker, versions: ExecutionVersioner) = {
- val cc = new CompletionChecker(conf, dirs, initVersioner)
+ val cc = new CompletionChecker(dirs, initVersioner)
visitAll(cc, initVersioner, plannedVertices)
(cc, new ExecutionVersioner(cc.completedVersions, initVersioner.nextVersion))
}
+
+ def getPackageVersions() = {
+ val packageFinder = new PackageFinder(conf, dirs, versions, cc.todo, workflow.packageDefs)
+ visitAll(packageFinder, versions, plannedVertices)
+ System.err.println("Found %d packages".format(packageFinder.packages.size))
+
+ err.println("Checking for already built packages...")
+ val packageVersions = new PackageVersioner(dirs, wd.versioners)
+ packageVersions.findAlreadyBuilt(packageFinder.packages.toSeq)
+ packageVersions
+ }
def list {
for(v: UnpackedWorkVert <- workflow.unpackedWalker(plannedVertices=plannedVertices).iterator) {
@@ -389,6 +400,7 @@ object Ducttape {
workflow.unpackedWalker(plannedVertices=plannedVertices).iterator.filter{v: UnpackedWorkVert => v.packed.value.name == goalTaskName}
}.toIterable
err.println("Found %d vertices with matching task name".format(matchingTasks.size))
+
var matchingReals: Iterable[RealTask] = {
matchingTasks.map{v: UnpackedWorkVert => {
val taskT: TaskTemplate = v.packed.value
@@ -401,8 +413,11 @@ object Ducttape {
}}.filter(_ != None).map(_.get)
}
err.println("Found %d vertices with matching realizations".format(matchingReals.size))
+
+ val packageVersions = getPackageVersions()
+
for(task: RealTask <- matchingReals) {
- val env = new TaskEnvironment(dirs, versions, task)
+ val env = new FullTaskEnvironment(dirs, versions, packageVersions, task)
for( (k,v) <- env.env) {
println("%s=%s".format(k,v))
}
@@ -439,13 +454,12 @@ object Ducttape {
def exec {
if(cc.todo.isEmpty) {
+ // TODO: Might need to re-run if any package versions have changed
err.println("All tasks to complete -- nothing to do")
} else {
err.println("Finding packages...")
- val packageFinder = new PackageFinder(conf, dirs, versions, cc.todo, workflow.packageDefs)
- visitAll(packageFinder, versions, plannedVertices)
- System.err.println("Found %d packages".format(packageFinder.packages.size))
-
+ val packageVersions = getPackageVersions()
+
err.println("Checking inputs...")
val inputChecker = new InputChecker(conf, dirs)
visitAll(inputChecker, versions, plannedVertices)
@@ -455,9 +469,11 @@ object Ducttape {
}
System.exit(1)
}
+
+ // TODO: Check package versions to see if any packages need rebuilding.
err.println("Work plan:")
- for(packageName <- packageFinder.packages) {
+ for(packageName <- packageVersions.packagesToBuild) {
err.println("%sBUILD:%s %s".format(conf.greenColor, conf.resetColor, packageName))
}
@@ -481,14 +497,14 @@ object Ducttape {
answer match {
case true => {
err.println("Retreiving code and building...")
- val builder = new PackageBuilder(conf, dirs, versions.workflowVersion)
- builder.build(packageFinder.packages)
+ val builder = new PackageBuilder(dirs, versions.workflowVersion, packageVersions)
+ builder.build(packageVersions.packagesToBuild)
// err.println("Removing partial output...")
// visitAll(new PartialOutputRemover(conf, dirs, versions, cc.partial), initVersioner, plannedVertices)
err.println("Executing tasks...")
try {
- visitAll(new Executor(conf, dirs, versions, workflow, plannedVertices, cc.completed, cc.todo), versions, plannedVertices, opts.jobs())
+ visitAll(new Executor(dirs, versions, packageVersions, workflow, plannedVertices, cc.completed, cc.todo), versions, plannedVertices, opts.jobs())
} catch {
case e: ExecutionException => {
err.println("%sERROR: %s%s".format(conf.errorColor, e.getMessage, conf.resetColor))
@@ -629,7 +645,7 @@ object Ducttape {
}
// TODO: Have run() function in each mode?
- opts.mode match {
+ ex2err(opts.mode match {
case "list" => list
case "env" => env
case "mark_done" => markDone
@@ -638,6 +654,6 @@ object Ducttape {
case "invalidate" => invalidate
case "purge" => purge
case _ => exec
- }
+ })
}
}
4 src/main/scala/ducttape/exec/BuildEnvironment.scala
View
@@ -1,8 +1,8 @@
package ducttape.exec
import java.io.File
-class BuildEnvironment(val dirs: DirectoryArchitect, val workflowVersion: Int, val packageName: String) {
- val buildDir = new File(dirs.assignBuildDir(packageName), workflowVersion.toString)
+class BuildEnvironment(val dirs: DirectoryArchitect, val packageVersion: String, val packageName: String) {
+ val buildDir = dirs.assignBuildDir(packageName, packageVersion)
val buildStdoutFile = new File(buildDir, "build_stdout.txt")
val buildStderrFile = new File(buildDir, "build_stderr.txt")
}
4 src/main/scala/ducttape/exec/CompletionChecker.scala
View
@@ -4,7 +4,6 @@ import java.io.File
import collection._
-import ducttape.Config
import ducttape.workflow.Realization
import ducttape.util.Files
import ducttape.util.OrderedSet
@@ -61,8 +60,7 @@ object CompletionChecker {
// the initVersioner is generally the MostRecentWorkflowVersioner, so that we can check if
// the most recent result is untouched, invalid, partial, or complete
-class CompletionChecker(conf: Config,
- dirs: DirectoryArchitect,
+class CompletionChecker(dirs: DirectoryArchitect,
initVersioner: WorkflowVersioner) extends UnpackedDagVisitor {
// we make a single pass to atomically determine what needs to be done
// so that we can then prompt the user for confirmation
11 src/main/scala/ducttape/exec/DirectoryArchitect.scala
View
@@ -29,8 +29,8 @@ class DirectoryArchitect(val workflowBaseDir: File, val confBaseDir: File) {
new File(unversionedDir, version.toString)
}
- def assignBuildDir(packageName: String): File = {
- new File(confBaseDir, ".packages/%s".format(packageName))
+ def assignBuildDir(packageName: String, packageVersion: String): File = {
+ new File(confBaseDir, ".packages/%s/%s".format(packageName, packageVersion))
}
def assignOutFile(spec: Spec, taskDef: TaskDef, realization: Realization, version: Int): File = {
@@ -86,4 +86,11 @@ class DirectoryArchitect(val workflowBaseDir: File, val confBaseDir: File) {
case _ => assignOutFile(srcSpec, srcTaskDef, srcRealization, srcVersion)
}
}
+
+ def getTempActionDir(actionName: String) = {
+ val f = File.createTempFile("ducttape", actionName)
+ f.delete() // delete file
+ f.mkdirs() // and make it a directory instead
+ f
+ }
}
10 src/main/scala/ducttape/exec/Executor.scala
View
@@ -2,7 +2,6 @@ package ducttape.exec
import collection._
-import ducttape.Config
import ducttape.versioner.WorkflowVersioner
import ducttape.util.Shell
import ducttape.util.Files
@@ -11,9 +10,9 @@ import ducttape.workflow.RealTask
import ducttape.workflow.HyperWorkflow
// workflow used for viz
-class Executor(conf: Config,
- dirs: DirectoryArchitect,
+class Executor(dirs: DirectoryArchitect,
versions: WorkflowVersioner,
+ packageVersioner: PackageVersioner,
workflow: HyperWorkflow,
plannedVertices: Set[(String,Realization)],
alreadyDone: Set[(String,Realization)],
@@ -33,7 +32,7 @@ class Executor(conf: Config,
override def visit(task: RealTask) {
if(todo((task.name, task.realization))) {
- val taskEnv = new TaskEnvironment(dirs, versions, task)
+ val taskEnv = new FullTaskEnvironment(dirs, versions, packageVersioner, task)
println("Running %s in %s".format(task.name, taskEnv.where.getAbsolutePath))
dirs.xdotFile.synchronized {
@@ -57,7 +56,8 @@ class Executor(conf: Config,
val exitCode = Shell.run(submitCommands, taskEnv.workDir, taskEnv.env, taskEnv.stdoutFile, taskEnv.stderrFile)
Files.write("%d".format(exitCode), taskEnv.exitCodeFile)
if(exitCode != 0) {
- println("%sTask %s/%s returned %s%s".format(conf.errorColor, task.name, task.realization.toString, exitCode, Console.RESET))
+ // TODO: Send message via listener?
+ println("Task %s/%s returned %s".format(task.name, task.realization.toString, exitCode))
failed += ((task.name, task.realization))
running -= ((task.name, task.realization))
dirs.xdotFile.synchronized {
25 src/main/scala/ducttape/exec/PackageBuilder.scala
View
@@ -1,25 +1,27 @@
package ducttape.exec
-
-import ducttape.Config
-import ducttape.workflow.hacks.Gimme
import ducttape.util.Shell
import ducttape.util.Files
import ducttape.syntax.AbstractSyntaxTree.PackageDef
-class PackageBuilder(conf: Config,
- dirs: DirectoryArchitect,
- workflowVersion: Int) {
+/**
+ * If we determine that a package is out of date (the requested version is not
+ * already built within ducttape), then we use this PackageBuilder to build
+ * a newly checked-out copy.
+ */
+class PackageBuilder(dirs: DirectoryArchitect,
+ workflowVersion: Int,
+ packageVersions: PackageVersioner) {
def build(packages: Iterable[PackageDef]) {
- for(myPackage <- packages) {
- val buildEnv = new BuildEnvironment(dirs, workflowVersion, myPackage.name)
- System.err.println("%sBuilding tools for: %s in %s%s".format(conf.taskColor, myPackage.name, buildEnv.buildDir, Console.RESET))
+ for(myPackage: PackageDef <- packages) {
+ val buildEnv = new BuildEnvironment(dirs, packageVersions(myPackage.name), myPackage.name)
+ System.err.println("Building tools %s in %s".format(myPackage.name, buildEnv.buildDir))
// TODO: XXX: Can build ever interfere with another running workflow?
if(buildEnv.buildDir.exists) {
System.err.println("Removing incomplete package build: %s".format(buildEnv.buildDir.toString))
Files.deleteDir(buildEnv.buildDir)
}
- buildEnv.buildDir.mkdirs()
+ packageVersions.checkout(myPackage, buildEnv.buildDir)
// TODO: XXX: Resolve the versioner and then get the checkout command
// TODO: Run static analysis on package code
@@ -32,7 +34,8 @@ class PackageBuilder(conf: Config,
val exitCode = Shell.run(buildCmds, buildEnv.buildDir, env, buildEnv.buildStdoutFile, buildEnv.buildStderrFile)
if(exitCode != 0) {
// just bail out, this workflow is doomed without its tools
- System.err.println("%sERROR: Build task %s returned %s%s".format(conf.errorColor, myPackage.name, exitCode, Console.RESET))
+ // TODO: Throw and then colorize output.
+ System.err.println("ERROR: Build task %s returned %s".format(myPackage.name, exitCode))
System.exit(1)
}
}
6 src/main/scala/ducttape/exec/PartialOutputRemover.scala
View
@@ -1,7 +1,5 @@
package ducttape.exec
-import ducttape.Config
-
import ducttape.versioner.WorkflowVersioner
import ducttape.workflow.Realization
import ducttape.workflow.RealTask
@@ -11,9 +9,9 @@ object PartialOutputRemover {
def hasPartialOutput(taskEnv: TaskEnvironment) = taskEnv.where.exists
}
-class PartialOutputRemover(conf: Config,
- dirs: DirectoryArchitect,
+class PartialOutputRemover(dirs: DirectoryArchitect,
versions: WorkflowVersioner,
+ packageVersions: PackageVersioner,
partial: Set[(String,Realization)]) extends UnpackedDagVisitor {
override def visit(task: RealTask) {
40 src/main/scala/ducttape/exec/TaskEnvironment.scala
View
@@ -3,10 +3,15 @@ package ducttape.exec
import ducttape.versioner.WorkflowVersioner
import ducttape.workflow.RealTask
import ducttape.workflow.Realization
-import ducttape.workflow.hacks.Gimme
import java.io.File
-class TaskEnvironment(val dirs: DirectoryArchitect, val versions: WorkflowVersioner, val task: RealTask) {
+/**
+ * Unlike the FullTaskEnvironment, does not require knowledge of packageVersions,
+ * but does not provide full list of environment variables
+ */
+class TaskEnvironment(val dirs: DirectoryArchitect,
+ val versions: WorkflowVersioner,
+ val task: RealTask) {
// grab input paths -- how are these to be resolved?
// If this came from a branch point, its source vertex *might* not
@@ -43,22 +48,33 @@ class TaskEnvironment(val dirs: DirectoryArchitect, val versions: WorkflowVersio
//err.println("For outSpec %s got path: %s".format(outSpec, outFile))
(outSpec.name, outFile.getAbsolutePath)
}
+
+ val where = dirs.assignDir(task.taskDef, task.realization, task.version)
+ val stdoutFile = new File(where, "stdout.txt")
+ val stderrFile = new File(where, "stderr.txt")
+ val workDir = new File(where, "work")
+ val exitCodeFile = new File(where, "exit_code.txt")
+ val invalidatedFile = new File(where, "INVALIDATED")
+}
+/**
+ * Includes all environment variables, but requires knowledge of packgeVersions
+ */
+class FullTaskEnvironment(dirs: DirectoryArchitect,
+ versions: WorkflowVersioner,
+ val packageVersions: PackageVersioner,
+ task: RealTask) extends TaskEnvironment(dirs, versions, task) {
val packageNames: Seq[String] = task.packages.map(_.name)
val packageBuilds: Seq[BuildEnvironment] = {
- packageNames.map(name => new BuildEnvironment(dirs, versions.workflowVersion, name))
+ packageNames.map{name => {
+ val packageVersion = packageVersions(name)
+ new BuildEnvironment(dirs, packageVersion, name)
+ }}
}
val packageEnvs: Seq[(String,String)] = {
- packageBuilds.map(build => (build.packageName, new File(build.buildDir, build.packageName).getAbsolutePath) )
+ packageBuilds.map(build => (build.packageName, build.buildDir.getAbsolutePath) )
}
// TODO: Add correct build paths to task env
lazy val env = inputs ++ outputs ++ params ++ packageEnvs
-
- val where = dirs.assignDir(task.taskDef, task.realization, task.version)
- val stdoutFile = new File(where, "stdout.txt")
- val stderrFile = new File(where, "stderr.txt")
- val workDir = new File(where, "work")
- val exitCodeFile = new File(where, "exit_code.txt")
- val invalidatedFile = new File(where, "INVALIDATED")
-}
+}
67 src/main/scala/ducttape/syntax/AbstractSyntaxTree.scala
View
@@ -6,9 +6,17 @@ import java.io.File
object AbstractSyntaxTree {
/** Parent class of all types representing elements in an abstract syntax tree. */
- abstract class ASTType extends Positional {}
+ trait ASTType extends Positional {
+ private var _file = new File("unknown_file")
+ def declaringFile_=(f: File) { _file = f }
+ def declaringFile: File = _file
+
+ // used for traversing AST... abstractly
+ def children: Seq[ASTType]
+ }
class Comments(val value: Option[String]) extends ASTType {
+ override def children = Nil
override def toString() = value match {
case Some(s:String) => s
case None => ""
@@ -16,29 +24,33 @@ object AbstractSyntaxTree {
}
/** Type of the right hand side in an assignment. */
- abstract class RValue extends ASTType;
+ trait RValue extends ASTType;
/** Unbound is the right hand side type in an underspecified variable declaration.
*
* Conceptually, Unbound can be thought of as the None case if one were to define Option[+RValue].
*/
case class Unbound() extends RValue {
+ override def children = Nil
override def toString() = ""
}
/** Type of a literal string value, in the right-hand side context of an assignment. */
case class Literal(val value: String) extends RValue {
+ override def children = Nil
override def toString() = "Literal='%s'".format(value)
}
/** Type of a variable reference, in the right-hand side context of an assignment. */
case class ConfigVariable(val value: String) extends RValue {
+ override def children = Nil
override def toString() = "$%s".format(value)
}
/** Type of a variable reference attached to a specific task,
* in the right-hand side context of an assignment. */
case class TaskVariable(val taskName: String, val value: String) extends RValue {
+ override def children = Nil
override def toString() = "$%s@%s".format(value,taskName)
}
@@ -50,6 +62,7 @@ object AbstractSyntaxTree {
val start: BigDecimal,
val end: BigDecimal,
val increment: BigDecimal) extends RValue {
+ override def children = Nil
override def toString() = {
branchPointName match {
case Some(s) => "(%s: %s..%s..%s)".format(s,start,end,increment)
@@ -61,6 +74,7 @@ object AbstractSyntaxTree {
/** Pair containing a branch point name and a branch name. */
class BranchGraftElement(val branchPointName: String,
val branchName: String) extends ASTType {
+ override def children = Nil
override def toString() = "%s:%s".format(branchPointName,branchName)
}
@@ -68,6 +82,7 @@ object AbstractSyntaxTree {
case class BranchGraft(val variableName: String,
val taskName: String,
val branchGraftElements: Seq[BranchGraftElement]) extends RValue {
+ override def children = branchGraftElements
override def toString() =
"$%s@%s%s".format(variableName,taskName,branchGraftElements.toString())
}
@@ -76,12 +91,12 @@ object AbstractSyntaxTree {
/**
* Abstract specification of a variable name and its right hand side.
*/
- class AbstractSpec[+RValue](val name: String, val rval: RValue, val dotVariable: Boolean) extends ASTType {
+ class AbstractSpec[+A <: RValue](val name: String, val rval: A, val dotVariable: Boolean) extends ASTType {
+ override def children = Seq(rval)
override def hashCode() = name.hashCode()
override def equals(that: Any) = that match { case other: AbstractSpec[_] => (other.name == this.name) }
override def toString() = "%s=%s".format(name, rval)
- }
-
+ }
type Spec = AbstractSpec[RValue]
type LiteralSpec = AbstractSpec[Literal]
@@ -89,21 +104,31 @@ object AbstractSyntaxTree {
* A key=value assignment defined in a "config" definition or config file.
*/
class ConfigAssignment(val spec: Spec, val comments: Comments) extends ASTType {
+ override def children = Seq(spec, comments)
override def toString() = spec.toString()
}
- abstract class Specs extends ASTType {
+ trait Specs extends ASTType {
val specs: Seq[Spec]
val comments: Comments
}
- case class TaskInputs(val specs: Seq[Spec], val comments: Comments) extends Specs;
- case class TaskOutputs(val specs: Seq[Spec], val comments: Comments) extends Specs;
- case class TaskParams(val specs: Seq[Spec], val comments: Comments) extends Specs;
- case class TaskPackageNames(val specs: Seq[Spec], val comments: Comments) extends Specs;
+ case class TaskInputs(val specs: Seq[Spec], val comments: Comments) extends Specs {
+ override def children = specs ++ Seq(comments)
+ }
+ case class TaskOutputs(val specs: Seq[Spec], val comments: Comments) extends Specs {
+ override def children = specs ++ Seq(comments)
+ }
+ case class TaskParams(val specs: Seq[Spec], val comments: Comments) extends Specs {
+ override def children = specs ++ Seq(comments)
+ }
+ case class TaskPackageNames(val specs: Seq[Spec], val comments: Comments) extends Specs {
+ override def children = specs ++ Seq(comments)
+ }
/** Branch in a hyperworkflow, defined in the right hand side of a variable declaration. */
case class BranchPointDef(val name: Option[String], val specs: Seq[Spec]) extends RValue {
+ override def children = specs
override def toString() = {
name match {
case Some(s) => "(%s: %s)".format(s, specs.mkString(" "))
@@ -114,12 +139,15 @@ object AbstractSyntaxTree {
/** Reference, in a plan, to a branchpoint and one or more of its branches. */
case class BranchPointRef(val name: String, val branchNames: Seq[String]) extends ASTType {
+ override def children = Nil
override def toString() = {
"(%s: %s)".format(name, branchNames.mkString(" "))
}
}
+ // NOTE: This has been replaced by the bash parser and BashCode
class ShellCommands(val value: String) extends ASTType {
+ override def children = Nil
override def toString() = value
}
@@ -128,11 +156,12 @@ object AbstractSyntaxTree {
// }
class TaskHeader(val specsList: List[Specs]) extends ASTType {
+ override def children = specsList
override def toString() = specsList.mkString(" ")
}
/** Defines a block of ducttape code, such as a task definition. */
- class Block extends ASTType;
+ trait Block extends ASTType;
/**
* Short for "TaskDefinition". Abbreviated due to its pervasiveness in the code.
@@ -144,6 +173,8 @@ object AbstractSyntaxTree {
val name: String,
val header: TaskHeader,
val commands: BashCode) extends Block {
+
+ override def children = Seq(comments, header, commands)
lazy val packageSpecList: Seq[TaskPackageNames] = header.specsList.collect{ case x: TaskPackageNames => x }
lazy val inputSpecList: Seq[TaskInputs] = header.specsList.collect{ case x: TaskInputs => x }
@@ -167,26 +198,31 @@ object AbstractSyntaxTree {
override def toString() = name
}
type PackageDef = TaskDef;
+ type ActionDef = TaskDef
class CallDefinition(val comments: Comments,
val name: String,
val header: TaskHeader,
val functionName: String) extends Block {
+ override def children = Seq(comments, header)
override def toString() = name
}
- class GroupDefinition(val comments:Comments,
+ class GroupDefinition(val comments: Comments,
val keyword: String,
val name: String,
val header: TaskHeader,
val blocks: Seq[Block]) extends Block {
+ override def children = Seq(comments, header) ++ blocks
override def toString() = name
}
+ type VersionerDef = GroupDefinition;
class ConfigDefinition(val keyword: String,
val comments: Comments,
val name: Option[String],
val lines: Seq[ConfigAssignment]) extends Block {
+ override def children = Seq(comments) ++ lines
override def toString() = {
name match {
case None => "GLOBAL"
@@ -196,6 +232,7 @@ object AbstractSyntaxTree {
}
class CrossProduct(val goals: Seq[String], val value: Seq[BranchPointRef]) extends ASTType {
+ override def children = value
override def toString() = {
"reach %s via %s".format(goals.mkString(","),value.mkString(" * "))
}
@@ -204,6 +241,7 @@ object AbstractSyntaxTree {
class PlanDefinition(val comments: Comments,
val name: Option[String],
val crossProducts: Seq[CrossProduct]) extends Block {
+ override def children = Seq(comments) ++ crossProducts
override def toString() = name match {
case None => "GLOBAL"
case Some(s: String) => s
@@ -212,8 +250,13 @@ object AbstractSyntaxTree {
/** Ducttape hyperworkflow file. */
class WorkflowDefinition(val file: File, val blocks: Seq[Block]) extends ASTType {
+ override def children = blocks
+
lazy val plans: Seq[PlanDefinition] = blocks.collect{case x: PlanDefinition => x}
+ private lazy val groupLikes: Seq[GroupDefinition] = blocks.collect{case x: GroupDefinition => x}
+ lazy val versioners: Seq[VersionerDef] = groupLikes.filter{x: GroupDefinition => x.keyword == "versioner"}
+
private lazy val configLikes: Seq[ConfigDefinition] = blocks.collect{case x: ConfigDefinition => x}
lazy val configs: Seq[ConfigDefinition] = configLikes.filter{t: ConfigDefinition => t.keyword == "config"}
lazy val globals: Seq[ConfigAssignment] = configLikes.filter{t: ConfigDefinition => t.keyword == "global"}.flatMap{ _.lines}
3  src/main/scala/ducttape/syntax/FileFormatException.scala
View
@@ -3,6 +3,9 @@ package ducttape.syntax
import java.io.File
import scala.util.parsing.input.Position
+/**
+ * Each element of ref has (file, line, col, untilLine)
+ */
class FileFormatException(val msg: String, val refs: Seq[(File, Int, Int, Int)]) extends Exception(msg) {
def this(msg: String, file: File, line: Int, col: Int) = this(msg, List( (file, line, col, line) ))
def this(msg: String, file: File, pos: Position) = this(msg, List( (file, pos.line, pos.column, pos.line) ))
12 src/main/scala/ducttape/syntax/Grammar.scala
View
@@ -2,20 +2,18 @@ package ducttape.syntax
import java.io.File
import java.math.BigDecimal
-
import org.apache.commons.lang3.StringEscapeUtils
-
import scala.util.parsing.combinator.Parsers
import scala.util.parsing.combinator.RegexParsers
import scala.util.parsing.input.CharArrayReader
import scala.util.parsing.input.Position
import scala.util.parsing.input.Positional
import scala.util.matching.Regex
-
import ducttape.syntax.AbstractSyntaxTree._
+import scala.util.parsing.input.NoPosition
object Grammar {
- import ducttape.syntax.GrammarParser._ // we need visibility of Parser, etc.
+ import ducttape.syntax.GrammarParser._ // we need visibility of Parser, etc.
/** End of line characters, including end of file. */
val eol: Parser[String] = literal("\r\n") | literal("\n") | regex("""\z""".r) | literal(CharArrayReader.EofCh.toString)
@@ -353,7 +351,7 @@ object Grammar {
literal(")"
)
) ^^ {
- case ((bpName: Option[String])~(start: BigDecimal)~(end: BigDecimal)~(Some(increment:BigDecimal))) =>
+ case ((bpName: Option[String])~(start: BigDecimal)~(end: BigDecimal)~(Some(increment: BigDecimal))) =>
new SequentialBranchPoint(bpName,start,end,increment)
case ((bpName: Option[String])~(start: BigDecimal)~(end: BigDecimal)~(None)) =>
new SequentialBranchPoint(bpName,start,end,new BigDecimal("1"))
@@ -478,8 +476,8 @@ object Grammar {
val branchAssignment: Parser[Spec] = positioned(
(basicAssignment("branch",failure(_),failure(_),failure(_)) | rvalue) ^^ {
- case assignment:Spec => assignment
- case rhs:RValue => new Spec(null,rhs,false)
+ case assignment: Spec => assignment
+ case rhs: RValue => new Spec(null,rhs,false)
}
)
15 src/main/scala/ducttape/syntax/GrammarParser.scala
View
@@ -15,12 +15,20 @@ object GrammarParser extends RegexParsers {
override val skipWhitespace = false;
+ private def addFileInfo(element: ASTType, file: File) {
+ element.declaringFile = file
+ element.children.foreach(addFileInfo(_, file))
+ }
+
def readWorkflow(file: File): WorkflowDefinition = {
val result: ParseResult[Seq[Block]] = parseAll(Grammar.blocks, IO.read(file, "UTF-8"))
val pos = result.next.pos
return result match {
- case Success(blocks: Seq[Block], _) => new WorkflowDefinition(file,blocks)
+ case Success(blocks: Seq[Block], _) => {
+ blocks.foreach(addFileInfo(_, file))
+ new WorkflowDefinition(file, blocks)
+ }
case Failure(msg, _) =>
throw new FileFormatException("ERROR: line %d column %d: %s".format(pos.line, pos.column, msg), file, pos)
case Error(msg, _) =>
@@ -33,7 +41,10 @@ object GrammarParser extends RegexParsers {
val pos = result.next.pos
return result match {
- case Success(asses: Seq[ConfigAssignment], _) => asses
+ case Success(asses: Seq[ConfigAssignment], _) => {
+ asses.foreach(addFileInfo(_, file))
+ asses
+ }
case Failure(msg, _) =>
throw new FileFormatException("ERROR: line %d column %d: %s".format(pos.line, pos.column, msg), file, pos)
case Error(msg, _) =>
5 src/main/scala/ducttape/syntax/bash.scala
View
@@ -5,8 +5,8 @@ import scala.util.parsing.combinator.RegexParsers
import scala.util.parsing.input.CharArrayReader
import scala.util.parsing.input.Position
import scala.util.parsing.input.Positional
-
import java.io.File
+import ducttape.syntax.AbstractSyntaxTree.ASTType
// TODO: Move this into unit tests
/*
@@ -73,7 +73,8 @@ function ohai {
*/
// TODO: Pass a StringBuilder down through the AST to make stringification faster
-class BashCode(val code: String, val vars: Set[String] = Set.empty) {
+class BashCode(val code: String, val vars: Set[String] = Set.empty) extends ASTType {
+ override def children = Nil // TODO: Name exactly what line vars come from
override def toString = code
}
20 src/main/scala/ducttape/workflow/hacks/Gimme.scala
View
@@ -1,20 +0,0 @@
-package ducttape.workflow.hacks
-
-import java.io.File
-
-object Gimme {
-
- def getPackagesFromParams(params: Seq[(String,String)]): Seq[String] = {
- params.filter{case (k:String, v:String) => k == ".gimme"}.map(_._2)
- }
-
- def getCommand(workflowDir: File, packageName: String): String = {
- val gimmeDir = new File(workflowDir, "gimme")
- val gimmeScript = new File(gimmeDir, packageName)
- if(!gimmeScript.exists) {
- throw new RuntimeException("Gimme script for package %s not found at %s".format(packageName, gimmeScript.getAbsolutePath))
- }
- "%s gimme %s".format(gimmeScript.getAbsolutePath, packageName)
- }
-
-}
7 syntax/tutorial/2-packages/1-disk.tape
View
@@ -12,14 +12,9 @@ package moses :: .versioner=disk .url="http://moses.svn.sourceforge.net/svnroot/
make -j16
}
-# TODO: Use proper rsync args
-# TODO: Disk versioner cannot support updating to a particular revision...
versioner disk :: url rev {
action checkout > dir {
- rsync $url $dir
- }
- action update {
- rsync $url $dir
+ cp $url $dir
}
action repo_version > version {
find $url -type f -printf '%TY-%Tm-%Td_%TT %p\n' | sort | awk 'END{print $1}' > $version
31 syntax/tutorial/2-packages/1-disk.tape.TODO
View
@@ -1,31 +0,0 @@
-# 1) Using package versioning without any source code control / versioning system
-# Use this only if git and svn are completely foreign
-# concepts to you. Beyond just getting started with ducttape,
-# use of this versioner is discouraged.
-
-task experiment_task: moses < in=/etc/passwd > out :: N=5 {
- $moses -version
-}
-
-package moses :: .versioner=disk .url=http://moses.svn.sourceforge.net/svnroot/moses .rev=NULL {
- ./configure
- make -j16
-}
-
-# TODO: Use proper rsync args
-# TODO: Disk versioner cannot support updating to a particular revision...
-versioner disk :: url rev {
- action checkout > dir {
- rsync $url $dir
- }
- action update {
- rsync $url $dir
- }
- action repo_version > version {
- find $url -type f -printf '%TY-%Tm-%Td_%TT %p\n' | sort | awk 'END{print $1}' > $version
- }
- action local_version > version date {
- find . -type f -printf '%TY-%Tm-%Td_%TT %p\n' | sort | awk 'END{print $1}' > $date
- cp $date $version
- }
-}
25 syntax/tutorial/2-packages/2-git.tape
View
@@ -4,14 +4,14 @@
# * To reproduce a workflow, you need to know what version of
# the software you ran
# * in, out, and N are shown only to illustrate syntax
-task param_step : codeheap < in=/etc/passwd > out :: N=5 {
- ls | $codeheap/do_stuff | tee $out
+task lunchtime : lunchpy {
+ $lunchpy/lunch.py Indian Mexican Italian
}
# * Build commands are only called when versioner indicates a new version
-package codeheap :: .versioner=git .url=/home/codeheap_git_repo {
- # We don't actually need to do this for python code,
- # but for the sake of example...
+package lunchpy :: .versioner=git .url="git://github.com/mjdenkowski/lunchpy.git" .ref=HEAD {
+ # We don't actually need to compile anything for python code,
+ # but for the sake of example we'll make this program run a bit faster
python -m compileall .
}
@@ -24,20 +24,17 @@ package codeheap :: .versioner=git .url=/home/codeheap_git_repo {
# * All other commands are run inside $dir
# * As we will see with inline branches, specializations such as checkout and update
# inherit all of their parent's parameters so that update has visibility of $url and $rev
-versioner git :: url rev {
+versioner git :: url ref {
action checkout > dir {
git clone $url $dir
}
- action update {
- git pull
- git checkout $rev
- }
action repo_version > version {
- git ls-remote origin HEAD | cut -f1 > $version
+ git ls-remote $url $ref | cut -f1 > $version
}
- # Used to confirm version after update/install
- action local_version > version date {
+ # TODO: Can we do without this? Just check repo version as we checkout? (potential race condition)
+ # Used to confirm version after checkout
+ action local_version > version date {
git rev-parse HEAD > $version
git log -1 | awk '/^Date/{$1=""; print}' > $date
}
-}
+}
43 syntax/tutorial/2-packages/2-git.tape.TODO
View
@@ -1,43 +0,0 @@
-# Using package versioning
-#
-# * During R&D, software often changes while a workflow is running
-# * To reproduce a workflow, you need to know what version of
-# the software you ran
-# * in, out, and N are shown only to illustrate syntax
-task param-step : codeheap < in=/etc/passwd > out :: N=5 {
- ls | $codeheap/do_stuff | tee $out
-}
-
-# * Build commands are only called when versioner indicates a new version
-package codeheap :: .versioner=git .url=/home/codeheap_git_repo {
- # We don't actually need to do this for python code,
- # but for the sake of example...
- python -m compileall .
-}
-
-#############################################################################
-# The following implementation of a git versioner is actually built-in and
-# automatically available to all workflows -- it is provided here for clarity
-#############################################################################
-
-# * Checkout is run in a sanboxed directory and $dir will be a subdirectory (makes using git easier)
-# * All other commands are run inside $dir
-# * As we will see with inline branches, specializations such as checkout and update
-# inherit all of their parent's parameters so that update has visibility of $url and $rev
-versioner git :: url rev {
- action checkout > dir {
- git clone $url $dir
- }
- action update {
- git pull
- git checkout $rev
- }
- action repo_version > version {
- git ls-remote origin HEAD | cut -f1 > $version
- }
- # Used to confirm version after update/install
- action local_version > version date {
- git rev-parse HEAD > $version
- git log -1 | awk '/^Date/{$1=""; print}' > $date
- }
-}
3  syntax/tutorial/2-packages/3-svn.tape
View
@@ -29,9 +29,6 @@ versioner svn :: url rev {
action checkout > dir {
svn co -r $rev $url .
}
- action update {
- svn up -r $rev
- }
action repo_version > version {
svn info $url | awk -F': ' '/Revision/{print $2}' > $version
svn info $url | awk -F': ' '/Date/{print}' > $date
44 syntax/tutorial/2-packages/3-svn.tape.TODO
View
@@ -1,44 +0,0 @@
-# 6) Using package versioning with SVN
-task experiment_task: moses < in=/etc/passwd > out :: N=5 {
- $moses -version
-}
-
-package moses :: .versioner=svn .url=http://moses.svn.sourceforge.net/svnroot/moses .rev=HEAD {
- ./configure
- make -j16
-}
-
-# TODO: Eventually, ducttape will support "bundling" a workflow to be shared and reproduced
-# by other researchers. This package versioning system allows us to just include
-# the workflow configuration and SVN/git repo URLs and build commands. Others
-# can then reproduce our experiments just by customizing the workflow for
-# their machine's resources.
-# TODO: Given a workflow bundle and the --reproduce option, ducttape should
-# checkout and build exactly the version of the code indicated by the bundle
-# for each task. It can accomplish this by overriding the .rev parameter
-# with the value required for reproduction rather than HEAD or some user choice.
-
-#############################################################################
-# The following implementation of a git versioner is actually built-in and
-# automatically available to all workflows -- it is provided here for clarity
-#############################################################################
-
-# * "checkout" is run in a sandbox directory
-# * All other commands are run inside $dir
-versioner svn :: url rev {
- action checkout > dir {
- svn co -r $rev $url .
- }
- action update {
- svn up -r $rev
- }
- action repo_version > version {
- svn info $url | awk -F': ' '/Revision/{print $2}' > $version
- svn info $url | awk -F': ' '/Date/{print}' > $date
- }
- # Used to confirm version after update/install
- action local_version > version date {
- svn info | awk -F': ' '/Revision/{print $2}' > $version
- svn info | awk -F': ' '/Date/{print}' > $date
- }
-}
20 syntax/tutorial/2-packages/gimme/cdec
View
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-repo=/home/jhclark/software/cdec
-cores=16
-
-mode=$1
-dest=$2
-
-$(dirname $0)/gimme_repo_git $mode $repo $dest
-
-case $mode in
- gimme)
- cd $dest
- echo >&2 "Building cdec using scons"
- scons -j $cores --with-boost=$HOME/prefix
- ;;
- *) ;;
-esac
24 syntax/tutorial/2-packages/gimme/gimme_repo_git
View
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-mode=$1
-
-case $mode in
- repo-version) repo=$2; cd git ls-remote $repo | head -n1 |cut -c1-7 ;;
- dest-version) dest=$2; cd $dest; git log --pretty=format:'%h' -n1 ;;
- gimme)
- repo=$2
- dest=$3
- echo >&2 "Fetching package from git repository $repo into $dest"
- # TODO: Better handling for existing repos (add remote, then pull?)
- # TODO: Properly parse --rev
- git clone $repo $dest
- if (( $# >= 4 )); then
- rev=$4
- git checkout $rev
- fi
- $0 dest-version $dest > $dest/gimme.rev
- ;;
- *) echo >&2 "Unrecognized mode: $mode"; exit 1 ;;
-esac
21 syntax/tutorial/2-packages/gimme/gimme_repo_svn
View
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-case $1 in
- repo-version) repo=$2; svn info $repo | awk '/^Revision/{print $2}' ;;
- dest-version) dest=$2; svnversion $dest ;;
- gimme)
- repo=$2
- dest=$3
- echo >&2 "Fetching package from svn repository $repo into $dest"
- # TODO: Better handling for existing? (e.g. svn up?)
- if (( $# >= 4 )); then
- rev=$4
- svn checkout -r $rev $repo $dest
- else
- svn checkout $repo $dest
- fi
- $0 dest-version $dest > $dest/gimme.rev
- ;;
- *) echo >&2 "Unrecognized mode: $mode"; exit 1 ;;
-esac
21 syntax/tutorial/2-packages/gimme/gimme_repo_symlink
View
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-mode=$1
-
-case $mode in
- repo-version) echo "WARNING:VERSION_UNSUPPORTED_BY_SYMLINK" ;;
- dest-version) echo "WARNING:VERSION_UNSUPPORTED_BY_SYMLINK" ;;
- gimme)
- repo=$2
- dest=$3
- # TODO: Better handling for existing repos (add remote, then pull?)
- ln -s $repo $dest
- if (( $# >= 4 )); then
- echo >&2 "Versioning not supported by the SYMLINK repo"
- exit 1
- fi
- $0 dest-version $dest > $dest/gimme.rev
- ;;
- *) echo >&2 "Unrecognized mode: $mode"; exit 1 ;;
-esac
18 syntax/tutorial/2-packages/gimme/multeval
View
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-mode=$1
-dest=$2
-
-repo=/home/jhclark/software/multeval
-
-$(dirname $0)/gimme_repo_git $mode $repo $dest
-
-case $mode in
- gimme)
- cd $dest
- ./get_deps.sh
- ant
- ;;
- *) ;;
-esac
17 syntax/tutorial/2-packages/gimme/sa_extract
View
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-repo=/home/jhclark/software/sa-extract
-
-mode=$1
-dest=$2
-
-$(dirname $0)/gimme_repo_symlink $mode $repo $dest
-
-case $mode in
- repo-version) ;;
- dest-version) ;;
- gimme) ;;
- *) ;;
-esac
21 syntax/tutorial/2-packages/gimme/srilm
View
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-repo=/home/jhclark/software/srilm
-cores=16
-
-$(dirname $0)/gimme_repo_symlink $@
-
-case $1 in
- repo-version) ;;
- dest-version) ;;
- gimme)
- dest=$3
- cd $dest
- make -j $cores SRILM=$PWD MACHINE_TYPE=i686-m64 MAKE_PIC=yes World
-
- # Overwrite version
- cp ~/workspace/srilm/RELEASE $dest/gimme.ver
- ;;
- *) ;;
-esac
5 syntax/tutorial/6-schedulers/1-shell.tape
View
@@ -0,0 +1,5 @@
+submitter sge :: COMMANDS /* the bash commands from some task */ {
+ action run {
+ $COMMANDS
+ }
+}
56 syntax/tutorial/6-schedulers/2-sge.tape
View
@@ -3,17 +3,57 @@
# and contains the "payload" of this task
# * This assumes that scheduler submission happens asynchronously
# and that we must poll the scheduler to learn when the job has completed
-submitter sge :: cmds vmem walltime {
+submitter sge :: cpus vmem walltime q /* these can be passed as parameters to each task: .cpus .vmem .walltime .q */
+ :: COMMANDS /* the bash commands from some task */
+ :: TASK REALIZATON CONFIGURATION /* variables passed by ducttape */ {
+ # TODO: We're still missing a few variables that typically get passed in.
+ # SGE: echo "#$-l vmem=$vmem" >> $wrapper
+ # SGE: echo "#-l walltime=$walltime" >> $wrapper
action wrap > wrapper {
- #$-l vmem=$vmem
- #$-l walltime=$walltime
- $cmds
+ echo "#PBS -S /bin/bash" >> $wrapper
+ echo "#PBS -q $q" >> $wrapper
+ echo "#PBS -l nodes=1:ppn=$cpus" >> $wrapper
+ echo "#PBS -l walltime=$walltime" >> $wrapper
+ echo "#PBS -j oe" >> $wrapper
+ echo "#PBS -o localhost:$PWD/job.out" >> $wrapper
+ echo "#PBS -N $CONFIGURATION/$TASK/$REALIZATION" >> $wrapper
+
+ # Bash flags aren't necessarily passed into the scheduler
+ # so we must re-initialize them
+ echo "set -e # stop on errors" >> $wrapper
+ echo "set -o pipefail # stop on pipeline errors" >> $wrapper
+ echo "set -u # stop on undeclared variables" >> $wrapper
+ echo "set -x # show each command as it is executed" >> $wrapper
+
+ # The current working directory will also be changed by most schedulers
+ echo "cd $PWD/work" >> $wrapper
+
+ echo "$COMMANDS" >> $wrapper
}
- action run < wrapper > id {
- qsub $wrapper > $id
+ action run < wrapper > jobid {
+ qsub $wrapper > $jobid
}
- action poll > done exit_code {
- TODO # insert the lengthy script needed to properly get the exit code here
+ action poll < jobid > done exit_code {
+ # Note: This is complicated by having to poll rather than being able to just use SGE's -sync
+ jobid=$(echo $jobid | cut -d. -f1) # Remove server name, if specified
+
+ if [[ "$jobid" == "" ]]; then
+ echo >&2 "ERROR: Empty job id. Did the job fail to submit?"
+ exit 1
+ fi
+
+ while true; do
+ # Use -alm to avoid costly queries to logs we don't even need
+ exit_status=$(tracejob -alm $jobid | awk '/Exit_status=-?[0-9]+/{print $4}' | cut -d= -f2)
+ if [[ "$exit_status" != "" ]]; then
+ echo >&2 "Job exited with status $exit_status"
+ if [[ -e job.out ]]; then
+ cat job.out
+ fi
+ exit $exit_status
+ fi
+ sleep 30
+ done
}
action status > short_status long_status {
qstat -f $id | awk '/job_state = Q/{print "queued"} /job_state = R/{print "running"}' > $status
65 syntax/tutorial/6-schedulers/2-sge.tape.TODO
View
@@ -1,65 +0,0 @@
-# * The resource parameter vmem can be specified as .vmem at task declarations
-# * The "cmds" parameter is inserted as a direct string replacement by ducttape
-# and contains the "payload" of this task
-# * This assumes that scheduler submission happens asynchronously
-# and that we must poll the scheduler to learn when the job has completed
-submitter sge :: cpus vmem walltime q /* these can be passed as parameters to each task: .cpus .vmem .walltime .q */
- :: COMMANDS /* the bash commands from some task */
- :: TASK REALIZATON CONFIGURATION /* variables passed by ducttape */ {
- # TODO: We're still missing a few variables that typically get passed in.
- # SGE: echo "#$-l vmem=$vmem" >> $wrapper
- # SGE: echo "#-l walltime=$walltime" >> $wrapper
- action wrap > wrapper {
- echo "#PBS -S /bin/bash" >> $wrapper
- echo "#PBS -q $q" >> $wrapper
- echo "#PBS -l nodes=1:ppn=$cpus" >> $wrapper
- echo "#PBS -l walltime=$walltime" >> $wrapper
- echo "#PBS -j oe" >> $wrapper
- echo "#PBS -o localhost:$PWD/job.out" >> $wrapper
- echo "#PBS -N $CONFIGURATION/$TASK/$REALIZATION" >> $wrapper
-
- # Bash flags aren't necessarily passed into the scheduler
- # so we must re-initialize them
- echo "set -e # stop on errors" >> $wrapper
- echo "set -o pipefail # stop on pipeline errors" >> $wrapper
- echo "set -u # stop on undeclared variables" >> $wrapper
- echo "set -x # show each command as it is executed" >> $wrapper
-
- # The current working directory will also be changed by most schedulers
- echo "cd $PWD/work" >> $wrapper
-
- echo "$COMMANDS" >> $wrapper
- }
- action run < wrapper > jobid {
- qsub $wrapper > $jobid
- }
- action poll < jobid > done exit_code {
- # Note: This is complicated by having to poll rather than being able to just use SGE's -sync
- jobid=$(echo $jobid | cut -d. -f1) # Remove server name, if specified
-
- if [[ "$jobid" == "" ]]; then
- echo >&2 "ERROR: Empty job id. Did the job fail to submit?"
- exit 1
- fi
-
- while true; do
- # Use -alm to avoid costly queries to logs we don't even need
- exit_status=$(tracejob -alm $jobid | awk '/Exit_status=-?[0-9]+/{print $4}' | cut -d= -f2)
- if [[ "$exit_status" != "" ]]; then
- echo >&2 "Job exited with status $exit_status"
- if [[ -e job.out ]]; then
- cat job.out
- fi
- exit $exit_status
- fi
- sleep 30
- done
- }
- action status > short_status long_status {
- qstat -f $id | awk '/job_state = Q/{print "queued"} /job_state = R/{print "running"}' > $status
- }
-}
-
-# NOTE: File transfers and decompression are handled by
-# a per-machine limit for each of these tasks and are
-# but are not included in the submit-script time
13 syntax/tutorial/6-schedulers/3-meta-config.XXX
View
@@ -0,0 +1,13 @@
+# Given a step like this:
+[g1.step2] a=$g1.step1/a b=$g1.step1/b > x y :: @@big
+ cat < $a > $x
+ cat < $b > $y
+
+#### TODO: Introduce these items after schedulers and meta-parameters are introduced...
+g1.step2.@vmem = 32g
+
+# We use the += operator to add parameter sets
+g1.step2 += @@big
+
+#### TODO: Introduce this material after we introduce hyperworkflows
+# TODO: Syntax for adding parameter sweeps from config file

No commit comments for this range

Something went wrong with that request. Please try again.