Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

odb: use regular upload() when the source filesystem is local #6365

Merged
merged 2 commits into from
Jul 27, 2021
Merged

odb: use regular upload() when the source filesystem is local #6365

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
da30d55
odb: use regular upload() when the source filesystem is local
isidentical Jul 27, 2021
6aa5b4f
support download_file too
isidentical Jul 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
61 changes: 34 additions & 27 deletions dvc/objects/db/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from copy import copy
from typing import TYPE_CHECKING, Optional

from dvc.fs.local import LocalFileSystem
from dvc.objects.errors import ObjectDBPermissionError, ObjectFormatError
from dvc.objects.file import HashFile
from dvc.progress import Tqdm
Expand Down Expand Up @@ -75,6 +76,38 @@ def get(self, hash_info: "HashInfo"):
hash_info,
)

def _add_file(self, from_fs, from_info, to_info, move):
self.makedirs(to_info.parent)
use_move = isinstance(from_fs, type(self.fs)) and move
try:
if use_move:
self.fs.move(from_info, to_info)
elif isinstance(from_fs, LocalFileSystem):
if not isinstance(from_info, from_fs.PATH_CLS):
from_info = from_fs.PATH_CLS(from_info)
self.fs.upload(from_info, to_info)
isidentical marked this conversation as resolved.
Show resolved Hide resolved
else:
with from_fs.open(from_info, mode="rb") as fobj:
self.fs.upload_fobj(fobj, to_info)
except OSError as exc:
# If the target file already exists, we are going to simply
# ignore the exception (#4992).
#
# On Windows, it is not always guaranteed that you'll get
# FileExistsError (it might be PermissionError or a bare OSError)
# but all of those exceptions raised from the original
# FileExistsError so we have a separate check for that.
if isinstance(exc, FileExistsError) or (
os.name == "nt"
and exc.__context__
and isinstance(exc.__context__, FileExistsError)
):
logger.debug("'%s' file already exists, skipping", to_info)
if use_move:
from_fs.remove(from_info)
else:
raise

def add(
self,
path_info: "AnyPath",
Expand All @@ -97,33 +130,7 @@ def add(

cache_info = self.hash_to_path_info(hash_info.value)
# using our makedirs to create dirs with proper permissions
self.makedirs(cache_info.parent)
use_move = isinstance(fs, type(self.fs)) and move
try:
if use_move:
self.fs.move(path_info, cache_info)
else:
with fs.open(path_info, mode="rb") as fobj:
self.fs.upload_fobj(fobj, cache_info)
except OSError as exc:
# If the target file already exists, we are going to simply
# ignore the exception (#4992).
#
# On Windows, it is not always guaranteed that you'll get
# FileExistsError (it might be PermissionError or a bare OSError)
# but all of those exceptions raised from the original
# FileExistsError so we have a separate check for that.
if isinstance(exc, FileExistsError) or (
os.name == "nt"
and exc.__context__
and isinstance(exc.__context__, FileExistsError)
):
logger.debug("'%s' file already exists, skipping", path_info)
if use_move:
fs.remove(path_info)
else:
raise

self._add_file(fs, path_info, cache_info, move)
try:
if verify:
self.check(hash_info, check_hash=True)
Expand Down