-
Notifications
You must be signed in to change notification settings - Fork 351
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3124 from broadinstitute/ks_cwl_dir
CWL updates to add directories and additional files.
- Loading branch information
Showing
51 changed files
with
1,802 additions
and
641 deletions.
There are no files selected for viewing
74 changes: 74 additions & 0 deletions
74
backend/src/main/scala/cromwell/backend/io/DirectoryFunctions.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package cromwell.backend.io | ||
|
|
||
| import cats.instances.list._ | ||
| import cats.syntax.traverse._ | ||
| import cats.syntax.validated._ | ||
| import common.validation.ErrorOr._ | ||
| import common.validation.Validation._ | ||
| import cromwell.backend.io.DirectoryFunctions._ | ||
| import cromwell.core.path.{Path, PathFactory} | ||
| import wom.expression.IoFunctionSet | ||
| import wom.values.{WomFile, WomGlobFile, WomMaybeListedDirectory, WomMaybePopulatedFile, WomSingleFile, WomUnlistedDirectory} | ||
|
|
||
| import scala.concurrent.Future | ||
|
|
||
| trait DirectoryFunctions extends IoFunctionSet with PathFactory { | ||
| override def listAllFilesUnderDirectory(dirPath: String): Future[Seq[String]] = { | ||
| temporaryImplListPaths(dirPath) | ||
| } | ||
|
|
||
| // TODO: WOM: WOMFILE: This will likely use a Tuple2(tar file, dir list file) for each dirPath. | ||
| private final def temporaryImplListPaths(dirPath: String): Future[Seq[String]] = { | ||
| val errorOrPaths = for { | ||
| dir <- validate(buildPath(ensureSlashed(dirPath))) | ||
| files <- listFiles(dir) | ||
| } yield files.map(_.pathAsString) | ||
| Future.fromTry(errorOrPaths.toTry(s"Error listing files under $dirPath")) | ||
| } | ||
| } | ||
|
|
||
| object DirectoryFunctions { | ||
| def ensureSlashed(dir: String): String = if (dir.endsWith("/")) dir else s"$dir/" | ||
|
|
||
| def listFiles(path: Path): ErrorOr[List[Path]] = { | ||
| if (path.isDirectory) { | ||
| for { | ||
| pathListing <- validate(path.list.toList) | ||
| pathsPerListing <- pathListing.traverse(listFiles) | ||
| } yield pathsPerListing.flatten | ||
| } else { | ||
| List(path).valid | ||
| } | ||
| } | ||
|
|
||
| def listWomSingleFiles(womFile: WomFile, pathFactory: PathFactory, pathPatcher: String => String): ErrorOr[List[WomSingleFile]] = { | ||
| def listWomSingleFiles(womFile: WomFile): ErrorOr[List[WomSingleFile]] = { | ||
| womFile match { | ||
| case womSingleFile: WomSingleFile => List(womSingleFile).valid | ||
|
|
||
| case womUnlistedDirectory: WomUnlistedDirectory => | ||
| val errorOrListPaths = listFiles(pathFactory.buildPath(ensureSlashed(pathPatcher(womUnlistedDirectory.value)))) | ||
| errorOrListPaths.map(_.map(path => WomSingleFile(path.pathAsString))) | ||
|
|
||
| case womMaybePopulatedFile: WomMaybePopulatedFile => | ||
| val allFiles: List[WomFile] = | ||
| womMaybePopulatedFile.valueOption.toList.map(WomSingleFile) ++ womMaybePopulatedFile.secondaryFiles | ||
| allFiles.traverse(listWomSingleFiles).map(_.flatten) | ||
|
|
||
| case w: WomMaybeListedDirectory => | ||
| (w.valueOption, w.listingOption) match { | ||
| case (None, None) => Nil.valid | ||
| case (Some(value), None) => listWomSingleFiles(WomUnlistedDirectory(value)) | ||
| case (None, Some(listing)) => listing.toList.traverse(listWomSingleFiles).map(_.flatten) | ||
| // TODO: WOM: WOMFILE: This is a special case where files from a different path are supposed to end up under the directory. If this implementation is correct, remove this TODO. | ||
| case (Some(_), Some(listing)) => listing.toList.traverse(listWomSingleFiles).map(_.flatten) | ||
| } | ||
| // TODO: WOM: WOMFILE: How did a glob get here? Should this link into glob functions to list the globs? | ||
|
|
||
| case _: WomGlobFile => s"Unexpected glob / unable to list glob files at this time: $womFile".invalidNel | ||
| } | ||
| } | ||
|
|
||
| listWomSingleFiles(womFile) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,23 @@ | ||
| package cwl | ||
|
|
||
| import shapeless._ | ||
| import cwl.command.StringCommandPart | ||
| import shapeless._ | ||
| import wom.CommandPart | ||
|
|
||
| object ArgumentToCommandPart extends Poly1 { | ||
| implicit def script = at[Expression] { CwlExpressionCommandPart.apply } | ||
| implicit def caseStringOrExpression: Case.Aux[StringOrExpression, CommandPart] = at { | ||
| _.fold(this) | ||
| } | ||
|
|
||
| implicit def caseExpression: Case.Aux[Expression, CommandPart] = at { | ||
| CwlExpressionCommandPart.apply | ||
| } | ||
|
|
||
| implicit def clb = at[CommandLineBinding] { CommandLineBindingCommandPart.apply } | ||
| implicit def caseString: Case.Aux[String, CommandPart] = at { | ||
| StringCommandPart.apply | ||
| } | ||
|
|
||
| implicit def string = at[String] { StringCommandPart.apply } | ||
| implicit def caseCommandLineBinding: Case.Aux[CommandLineBinding, CommandPart] = at { | ||
| CommandLineBindingCommandPart.apply | ||
| } | ||
| } |
Oops, something went wrong.