From 4e876dd2d8b19d229f37fd775bcc369625f50ff8 Mon Sep 17 00:00:00 2001 From: pawel Date: Mon, 11 Nov 2019 16:13:13 +0100 Subject: [PATCH 1/3] import: intercept and rephrase OutputNotFound message --- dvc/exceptions.py | 9 +++++++++ dvc/repo/imp.py | 9 ++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 07a0ce8061..52b363c848 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -328,3 +328,12 @@ def __init__(self, target_infos): class CollectCacheError(DvcException): pass + + +class NoOutputInExternalRepoError(DvcException): + def __init__(self, repo_url, path): + super(NoOutputInExternalRepoError, self).__init__( + "Output: '{}' not found in target repository: '{}'".format( + path, repo_url + ) + ) diff --git a/dvc/repo/imp.py b/dvc/repo/imp.py index 68ead389b6..869039266a 100644 --- a/dvc/repo/imp.py +++ b/dvc/repo/imp.py @@ -1,6 +1,13 @@ +from dvc.exceptions import NoOutputInExternalRepoError +from dvc.exceptions import OutputNotFoundError + + def imp(self, url, path, out=None, rev=None): erepo = {"url": url} if rev is not None: erepo["rev"] = rev - return self.imp_url(path, out=out, erepo=erepo, locked=True) + try: + return self.imp_url(path, out=out, erepo=erepo, locked=True) + except OutputNotFoundError: + raise NoOutputInExternalRepoError(url, path) From e05f14f9db0c1ff78db5b9b4bc71012257b322c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Redzy=C5=84ski?= Date: Tue, 12 Nov 2019 12:18:39 +0100 Subject: [PATCH 2/3] import/get: delegate OutputNotFound handling to external repo context manager --- dvc/dependency/repo.py | 6 +++--- dvc/exceptions.py | 3 ++- dvc/external_repo.py | 6 +++++- dvc/repo/__init__.py | 12 ++++++++---- dvc/repo/fetch.py | 2 +- dvc/repo/get.py | 2 +- dvc/repo/imp.py | 9 +-------- 7 files changed, 21 insertions(+), 19 deletions(-) diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 20439c024e..68cd044a1d 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -49,10 +49,10 @@ def _make_repo(self, **overrides): def status(self): with self._make_repo() as repo: - current = repo.find_out_by_relpath(self.def_path).info + current = repo.find_out_by_path(self.def_path).info with self._make_repo(rev_lock=None) as repo: - updated = repo.find_out_by_relpath(self.def_path).info + updated = repo.find_out_by_path(self.def_path).info if current != updated: return {str(self): "update available"} @@ -71,7 +71,7 @@ def fetch(self): ) as repo: self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev() - out = repo.find_out_by_relpath(self.def_path) + out = repo.find_out_by_path(self.def_path) with repo.state: repo.cloud.pull(out.get_used_cache()) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 52b363c848..53a97905e1 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -60,9 +60,10 @@ class OutputNotFoundError(DvcException): """ def __init__(self, output): + self.failed_output = output super(OutputNotFoundError, self).__init__( "unable to find DVC-file with output '{path}'".format( - path=relpath(output) + path=relpath(self.failed_output) ) ) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index ccdadf6e6b..369099eb81 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -7,6 +7,7 @@ from funcy import retry +from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError from dvc.utils import remove @@ -19,7 +20,10 @@ def external_repo(url=None, rev=None, rev_lock=None, cache_dir=None): path = _external_repo(url=url, rev=rev_lock or rev, cache_dir=cache_dir) repo = Repo(path) - yield repo + try: + yield repo + except OutputNotFoundError as e: + raise NoOutputInExternalRepoError(url, e.failed_output) repo.close() diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index e0003b8087..d3f4a1a22d 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -418,10 +418,15 @@ def stages(self): return get_stages(self.graph) def find_outs_by_path(self, path, outs=None, recursive=False): + abs_path = ( + os.path.join(self.root_dir, path) + if not os.path.isabs(path) + else path + ) + if not outs: outs = [out for stage in self.stages for out in stage.outs] - abs_path = os.path.abspath(path) is_dir = self.tree.isdir(abs_path) def func(out): @@ -439,9 +444,8 @@ def func(out): return matched - def find_out_by_relpath(self, relpath): - path = os.path.join(self.root_dir, relpath) - out, = self.find_outs_by_path(path) + def find_out_by_path(self, relpath): + out, = self.find_outs_by_path(relpath) return out def is_dvc_internal(self, path): diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 531fd4d535..0440dddaea 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -81,7 +81,7 @@ def _fetch_external(self, repo_url, repo_rev, files): cache = NamedCache() for name in files: try: - out = repo.find_out_by_relpath(name) + out = repo.find_out_by_path(name) except OutputNotFoundError: failed += 1 logger.exception( diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 0117006840..09426caa58 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -49,7 +49,7 @@ def get(url, path, out=None, rev=None): # the same cache file might be used a few times in a directory. repo.cache.local.cache_types = ["reflink", "hardlink", "copy"] - o = repo.find_out_by_relpath(path) + o = repo.find_out_by_path(path) with repo.state: repo.cloud.pull(o.get_used_cache()) o.path_info = PathInfo(os.path.abspath(out)) diff --git a/dvc/repo/imp.py b/dvc/repo/imp.py index 869039266a..68ead389b6 100644 --- a/dvc/repo/imp.py +++ b/dvc/repo/imp.py @@ -1,13 +1,6 @@ -from dvc.exceptions import NoOutputInExternalRepoError -from dvc.exceptions import OutputNotFoundError - - def imp(self, url, path, out=None, rev=None): erepo = {"url": url} if rev is not None: erepo["rev"] = rev - try: - return self.imp_url(path, out=out, erepo=erepo, locked=True) - except OutputNotFoundError: - raise NoOutputInExternalRepoError(url, path) + return self.imp_url(path, out=out, erepo=erepo, locked=True) From 6736ff5808d5b7ba5a7d872d1fbda1f28edc338d Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 12 Nov 2019 11:22:39 +0000 Subject: [PATCH 3/3] Restyled by reorder-python-imports --- dvc/external_repo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 369099eb81..883ae3c082 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -7,7 +7,8 @@ from funcy import retry -from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError +from dvc.exceptions import NoOutputInExternalRepoError +from dvc.exceptions import OutputNotFoundError from dvc.utils import remove