-
Notifications
You must be signed in to change notification settings - Fork 598
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enhanced Google pipelines file copy strategy
Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
- Loading branch information
1 parent
4a167e7
commit 678405b
Showing
6 changed files
with
151 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
69 changes: 69 additions & 0 deletions
69
modules/nf-google/src/test/nextflow/cloud/google/GoogleSpecification.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| * Copyright 2013-2019, Centre for Genomic Regulation (CRG) | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package nextflow.cloud.google | ||
|
|
||
| import spock.lang.Specification | ||
|
|
||
| import java.nio.file.FileSystem | ||
| import java.nio.file.Path | ||
| import java.nio.file.attribute.BasicFileAttributes | ||
| import java.nio.file.spi.FileSystemProvider | ||
|
|
||
| /** | ||
| * | ||
| * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com> | ||
| */ | ||
| abstract class GoogleSpecification extends Specification { | ||
|
|
||
| protected Path mockGsPath(String path, boolean isDir=false) { | ||
| assert path.startsWith('gs://') | ||
|
|
||
| def tokens = path.tokenize('/') | ||
| def bucket = tokens[1] | ||
| def file = '/' + tokens[2..-1].join('/') | ||
|
|
||
| def attr = Mock(BasicFileAttributes) | ||
| attr.isDirectory() >> isDir | ||
| attr.isRegularFile() >> !isDir | ||
| attr.isSymbolicLink() >> false | ||
|
|
||
| def provider = Mock(FileSystemProvider) | ||
| provider.getScheme() >> 'gs' | ||
| provider.readAttributes(_, _, _) >> attr | ||
|
|
||
| def fs = Mock(FileSystem) | ||
| fs.provider() >> provider | ||
| fs.toString() >> ('gs:/' + bucket) | ||
| def uri = GroovyMock(URI) | ||
| uri.toString() >> path | ||
|
|
||
|
|
||
| def result = GroovyMock(Path) | ||
| result.bucket() >> bucket | ||
| result.toUriString() >> path | ||
| result.toString() >> file | ||
| result.getFileSystem() >> fs | ||
| result.toUri() >> uri | ||
| result.resolve(_) >> { mockGsPath("$path/${it[0]}")} | ||
| result.toAbsolutePath() >> result | ||
| result.asBoolean() >> true | ||
| return result | ||
| } | ||
|
|
||
|
|
||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
49 changes: 49 additions & 0 deletions
49
modules/nf-google/src/test/nextflow/extension/FilesExTest2.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| /* | ||
| * Copyright 2013-2019, Centre for Genomic Regulation (CRG) | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package nextflow.extension | ||
|
|
||
| import spock.lang.Specification | ||
| import spock.lang.Unroll | ||
|
|
||
| import java.nio.file.Path | ||
|
|
||
| import com.google.cloud.storage.contrib.nio.CloudStoragePath | ||
|
|
||
| /** | ||
| * | ||
| * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com> | ||
| */ | ||
| class FilesExTest2 extends Specification { | ||
|
|
||
| @Unroll | ||
| def 'should return uri string for #PATH' () { | ||
|
|
||
| when: | ||
| def path = PATH as Path | ||
| then: | ||
| path instanceof CloudStoragePath | ||
| FilesEx.toUriString(path) == PATH | ||
|
|
||
| where: | ||
| PATH | _ | ||
| 'gs://foo/bar' | _ | ||
| 'gs://foo' | _ | ||
| 'gs://foo/' | _ | ||
| 'gs://foo/bar/baz' | _ | ||
| 'gs://foo/bar/baz/' | _ | ||
| } | ||
| } |