Skip to content
Permalink
Browse files
fix: issue with duplicated prefix for checkpoints on cloud (#1294)
* create tests for issue #574
to reproduce a problem with the use of checkpoints in cloud environments

* do not add default remote prefix to files in checkpoints
  • Loading branch information
aryarm committed Feb 21, 2022
1 parent fd5daae commit 8ed0c8cb453b6ebf6df138391a0681ffc8442e09
Show file tree
Hide file tree
Showing 6 changed files with 57 additions and 1 deletion.
@@ -767,7 +767,7 @@ def _apply_wildcards(
is_unpack=is_unpack,
**aux_params
)
if apply_path_modifier:
if apply_path_modifier and not incomplete:
item = self.apply_path_modifier(item, property=property)

if is_unpack and not incomplete:
@@ -0,0 +1,32 @@
import os

# This test file is adapted from this one:
# https://github.com/snakemake/snakemake/blob/758fabdb64255f8ca79e9c1483ceab67eb39ff07/tests/test_google_lifesciences/Snakefile
from snakemake.remote.GS import RemoteProvider as GSRemoteProvider
GS = GSRemoteProvider()

rule all:
input:
"landsat-data.txt.bz2"

checkpoint copy:
input:
GS.remote("gcp-public-data-landsat/LC08/01/001/003/LC08_L1GT_001003_20170430_20170501_01_RT/LC08_L1GT_001003_20170430_20170501_01_RT_MTL.txt")
output:
"landsat-data.txt"
resources:
mem_mb=100
run:
shell("cp {input} {output}")

rule pack:
input:
lambda wildcards: checkpoints.copy.get().output[0]
output:
"landsat-data.txt.bz2"
conda:
"env.yml"
log:
"logs/pack.log"
shell:
"bzip2 -c {input} > {output}; echo successful > {log}"
@@ -0,0 +1 @@
{"message": "hahaha"}
@@ -0,0 +1,4 @@
channels:
- conda-forge
dependencies:
- bzip2
Empty file.
@@ -90,3 +90,22 @@ def test_touch_remote_prefix():
)
finally:
cleanup_google_storage(storage_prefix, bucket_name)


@google_credentials
def test_cloud_checkpoints_issue574():
"""see Github issue #574"""
bucket_name = "snakemake-testing-%s" % next(tempfile._get_candidate_names())
create_google_storage(bucket_name)
storage_prefix = "test_cloud_checkpoints_issue574"
workdir = dpath("test_cloud_checkpoints_issue574")
try:
run(
workdir,
use_conda=True,
default_remote_prefix="%s/%s" % (bucket_name, storage_prefix),
google_lifesciences=True,
google_lifesciences_cache=False,
)
finally:
cleanup_google_storage(storage_prefix, bucket_name)

0 comments on commit 8ed0c8c

Please sign in to comment.