Skip to content
Permalink
Browse files
feat: support XRootD as a default remote provider (#1017)
* Add the remote provider XRootD to the list of allowed default remote providers.

* see what happens if we set supports_default = True for XRootD

* make sure non-relative path is supplied to xrootd

* ensure paths are always non-relative

* missing slash

* snakemake expects copying to overwrite the destination file if it already exists

* Formatting.

* Remove extraneous forward-slashes

Co-authored-by: Johannes Köster <johannes.koester@uni-due.de>
  • Loading branch information
ryuwd and johanneskoester committed Aug 17, 2021
1 parent 8892bf2 commit fe03157c31210984fce53c35d5fb87b20d278fe7
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 3 deletions.
@@ -1931,6 +1931,7 @@ def get_argument_parser(profile=None):
"gridftp",
"iRODS",
"AzBlob",
"XRootD",
],
help="Specify default remote provider to be used for "
"all input and output files that don't yet specify "
@@ -21,6 +21,8 @@


class RemoteProvider(AbstractRemoteProvider):
supports_default = True

def __init__(
self, *args, keep_local=False, stay_on_remote=False, is_default=False, **kwargs
):
@@ -29,7 +31,7 @@ def __init__(
keep_local=keep_local,
stay_on_remote=stay_on_remote,
is_default=is_default,
**kwargs
**kwargs,
)

self._xrd = XRootDHelper()
@@ -59,7 +61,7 @@ def __init__(
keep_local=keep_local,
stay_on_remote=stay_on_remote,
provider=provider,
**kwargs
**kwargs,
)

if provider:
@@ -131,6 +133,12 @@ def _parse_url(self, url):
dirname, filename = os.path.split(match.group("path"))
# We need a trailing / to keep XRootD happy
dirname += "/"

# and also make sure we supply a non-relative path
# (snakemake removes double-slash // characters)
if not dirname.startswith("/"):
dirname = "/" + dirname

return domain, dirname, filename

def exists(self, url):
@@ -188,18 +196,24 @@ def copy(self, source, destination):
# Prepare the source path for XRootD
if not self._parse_url(source):
source = abspath(source)
else:
domain, dirname, filename = self._parse_url(source)
source = f"{domain}/{dirname}/{filename}"

# Prepare the destination path for XRootD
assert os.path.basename(source) == os.path.basename(destination)
if self._parse_url(destination):
domain, dirname, filename = self._parse_url(destination)
destination = f"{domain}/{dirname}/{filename}"
self.makedirs(domain, dirname)
else:
destination = abspath(destination)
if not os.path.isdir(os.path.dirname(destination)):
os.makedirs(os.path.dirname(destination))

# Perform the copy operation
process = client.CopyProcess()
process.add_job(source, destination)
process.add_job(source, destination, force=True)
process.prepare()
status, returns = process.run()
if not status.ok or not returns[0]["status"].ok:

0 comments on commit fe03157

Please sign in to comment.