Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions dvc/cachemgr.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def _get_odb(
cls, config, fs_path = get_cloud_fs(repo.config, **settings)
fs = fs or cls(**config)
if prefix:
fs_path = fs.path.join(fs_path, *prefix)
fs_path = fs.join(fs_path, *prefix)
if hash_name:
config["hash_name"] = hash_name
return get_odb(fs, fs_path, state=repo.state, **config)
Expand Down Expand Up @@ -86,7 +86,7 @@ def fs_cache(self):
return FileStorage(
key=(),
fs=self.local.fs,
path=self.local.fs.path.join(self.default_local_cache_dir, self.FS_DIR),
path=self.local.fs.join(self.default_local_cache_dir, self.FS_DIR),
)

def _init_odb(self, schemes):
Expand Down
2 changes: 1 addition & 1 deletion dvc/commands/dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def _collect_targets(repo, target, outs):
targets.extend([str(out) for out in stage.outs])
continue

for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)):
for out in outs_trie.itervalues(prefix=repo.fs.parts(path)):
targets.extend(str(out))

return targets
Expand Down
16 changes: 8 additions & 8 deletions dvc/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

if TYPE_CHECKING:
from dvc.fs import FileSystem
from dvc.types import DictStrAny, StrPath
from dvc.types import DictStrAny

logger = logger.getChild(__name__)

Expand Down Expand Up @@ -89,8 +89,8 @@ class Config(dict):

def __init__(
self,
dvc_dir: Optional["StrPath"] = None,
local_dvc_dir: Optional["StrPath"] = None,
dvc_dir: Optional[str] = None,
local_dvc_dir: Optional[str] = None,
validate: bool = True,
fs: Optional["FileSystem"] = None,
config: Optional["DictStrAny"] = None,
Expand All @@ -105,7 +105,7 @@ def __init__(
self.fs = fs or self.wfs

if dvc_dir:
self.dvc_dir = self.fs.path.abspath(dvc_dir)
self.dvc_dir = self.fs.abspath(dvc_dir)

self.local_dvc_dir = local_dvc_dir
if not fs and not local_dvc_dir:
Expand Down Expand Up @@ -145,10 +145,10 @@ def files(self) -> Dict[str, str]:
}

if self.dvc_dir is not None:
files["repo"] = self.fs.path.join(self.dvc_dir, self.CONFIG)
files["repo"] = self.fs.join(self.dvc_dir, self.CONFIG)

if self.local_dvc_dir is not None:
files["local"] = self.wfs.path.join(self.local_dvc_dir, self.CONFIG_LOCAL)
files["local"] = self.wfs.join(self.local_dvc_dir, self.CONFIG_LOCAL)

return files

Expand Down Expand Up @@ -302,11 +302,11 @@ def _to_relpath(conf_dir, path):
return path.def_path

if os.path.expanduser(path) != path:
return localfs.path.as_posix(path)
return localfs.as_posix(path)

if isinstance(path, RelPath) or not os.path.isabs(path):
path = relpath(path, conf_dir)
return localfs.path.as_posix(path)
return localfs.as_posix(path)

return path

Expand Down
6 changes: 2 additions & 4 deletions dvc/data_cloud.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,9 @@ def odb(self) -> "HashFileDB":

path = self.path
if self.worktree:
path = self.fs.path.join(
path, ".dvc", CacheManager.FILES_DIR, DEFAULT_ALGORITHM
)
path = self.fs.join(path, ".dvc", CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
else:
path = self.fs.path.join(path, CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
path = self.fs.join(path, CacheManager.FILES_DIR, DEFAULT_ALGORITHM)
return get_odb(self.fs, path, hash_name=DEFAULT_ALGORITHM, **self.config)

@cached_property
Expand Down
8 changes: 4 additions & 4 deletions dvc/dependency/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def workspace_status(self) -> Dict[str, str]:
if self.fs.version_aware:
old_fs_path = self.fs_path
try:
self.fs_path = self.fs.path.version_path(self.fs_path, None)
self.fs_path = self.fs.version_path(self.fs_path, None)
if self.changed_meta():
return {str(self): "update available"}
finally:
Expand All @@ -42,17 +42,17 @@ def workspace_status(self) -> Dict[str, str]:

def update(self, rev=None):
if self.fs.version_aware:
self.fs_path = self.fs.path.version_path(self.fs_path, rev)
self.fs_path = self.fs.version_path(self.fs_path, rev)
self.meta = self.get_meta()
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
self.fs_path = self.fs.version_path(self.fs_path, self.meta.version_id)

def download(self, to, jobs=None):
fs_download(self.fs, self.fs_path, to.fs_path, jobs=jobs)

def save(self):
super().save()
if self.fs.version_aware:
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
self.fs_path = self.fs.version_path(self.fs_path, self.meta.version_id)

def dumpd(self, **kwargs):
if self.fs.version_aware:
Expand Down
2 changes: 1 addition & 1 deletion dvc/dvcfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ def lockfile_contents(self) -> Dict[str, Any]:
def resolver(self) -> "DataResolver":
from .parsing import DataResolver

wdir = self.repo.fs.path.parent(self.path)
wdir = self.repo.fs.parent(self.path)
return DataResolver(self.repo, wdir, self.contents)

@cached_property
Expand Down
10 changes: 3 additions & 7 deletions dvc/fs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
ConfigError,
RemoteMissingDepsError,
)
from dvc_objects.fs.path import Path # noqa: F401

from .callbacks import Callback # noqa: F401
from .data import DataFileSystem # noqa: F401
Expand Down Expand Up @@ -54,23 +53,20 @@ def download(
from .callbacks import TqdmCallback

with TqdmCallback(
desc=f"Downloading {fs.path.name(fs_path)}",
desc=f"Downloading {fs.name(fs_path)}",
unit="files",
) as cb:
# NOTE: We use dvc-objects generic.copy over fs.get since it makes file
# download atomic and avoids fsspec glob/regex path expansion.
if fs.isdir(fs_path):
from_infos = [
path
for path in fs.find(fs_path)
if not path.endswith(fs.path.flavour.sep)
path for path in fs.find(fs_path) if not path.endswith(fs.flavour.sep)
]
if not from_infos:
localfs.makedirs(to, exist_ok=True)
return 0
to_infos = [
localfs.path.join(to, *fs.path.relparts(info, fs_path))
for info in from_infos
localfs.join(to, *fs.relparts(info, fs_path)) for info in from_infos
]
else:
from_infos = [fs_path]
Expand Down
3 changes: 3 additions & 0 deletions dvc/fs/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ def fs(

return _DataFileSystem(**self.fs_args)

def getcwd(self):
return self.fs.getcwd()

def isdvc(self, path, **kwargs):
return self.fs.isdvc(path, **kwargs)

Expand Down
81 changes: 57 additions & 24 deletions dvc/fs/dvc.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

from dvc.log import logger
from dvc_objects.fs.base import FileSystem
from dvc_objects.fs.path import Path

from .data import DataFileSystem

Expand Down Expand Up @@ -71,7 +70,7 @@ def _merge_info(repo, key, fs_info, dvc_info):


def _get_dvc_path(dvc_fs, subkey):
return dvc_fs.path.join(*subkey) if subkey else ""
return dvc_fs.join(*subkey) if subkey else ""


class _DVCFileSystem(AbstractFileSystem):
Expand Down Expand Up @@ -145,7 +144,51 @@ def __init__( # noqa: PLR0913
"remote_config": remote_config,
}

self.path = Path(self.sep, getcwd=self._getcwd)
def getcwd(self):
relparts: Tuple[str, ...] = ()
assert self.repo is not None
if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir):
relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir)
return self.root_marker + self.sep.join(relparts)

@classmethod
def join(cls, *parts: str) -> str:
return posixpath.join(*parts)

@classmethod
def parts(cls, path: str) -> Tuple[str, ...]:
ret = []
while True:
path, part = posixpath.split(path)

if part:
ret.append(part)
continue

if path:
ret.append(path)

break

ret.reverse()

return tuple(ret)

def normpath(self, path: str) -> str:
return posixpath.normpath(path)

def abspath(self, path: str) -> str:
if not posixpath.isabs(path):
path = self.join(self.getcwd(), path)
return self.normpath(path)

def relpath(self, path: str, start: Optional[str] = None) -> str:
if start is None:
start = "."
return posixpath.relpath(self.abspath(path), start=self.abspath(start))

def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]:
return self.parts(self.relpath(path, start=start))

@functools.cached_property
def repo(self):
Expand All @@ -170,15 +213,6 @@ def repo_factory(self):

return self.repo._fs_conf["repo_factory"]

def _getcwd(self):
relparts: Tuple[str, ...] = ()
assert self.repo is not None
if self.repo.fs.path.isin(self.repo.fs.path.getcwd(), self.repo.root_dir):
relparts = self.repo.fs.path.relparts(
self.repo.fs.path.getcwd(), self.repo.root_dir
)
return self.root_marker + self.sep.join(relparts)

@functools.cached_property
def fsid(self) -> str:
from fsspec.utils import tokenize
Expand All @@ -192,7 +226,7 @@ def fsid(self) -> str:

def _get_key(self, path: "StrPath") -> Key:
path = os.fspath(path)
parts = self.repo.fs.path.relparts(path, self.repo.root_dir)
parts = self.repo.fs.relparts(path, self.repo.root_dir)
if parts == (os.curdir,):
return ()
return parts
Expand All @@ -210,13 +244,13 @@ def _subrepos_trie(self):

def _get_key_from_relative(self, path) -> Key:
path = self._strip_protocol(path)
parts = self.path.relparts(path, self.root_marker)
parts = self.relparts(path, self.root_marker)
if parts and parts[0] == os.curdir:
return parts[1:]
return parts

def _from_key(self, parts: Key) -> str:
return self.repo.fs.path.join(self.repo.root_dir, *parts)
return self.repo.fs.join(self.repo.root_dir, *parts)

@functools.cached_property
def _datafss(self):
Expand Down Expand Up @@ -282,7 +316,7 @@ def _is_dvc_repo(self, dir_path):

from dvc.repo import Repo

repo_path = self.repo.fs.path.join(dir_path, Repo.DVC_DIR)
repo_path = self.repo.fs.join(dir_path, Repo.DVC_DIR)
return self.repo.fs.isdir(repo_path)

def _get_subrepo_info(
Expand Down Expand Up @@ -336,7 +370,7 @@ def ls(self, path, detail=True, dvc_only=False, **kwargs):
dvc_path = _get_dvc_path(dvc_fs, subkey)
with suppress(FileNotFoundError, NotADirectoryError):
for info in dvc_fs.ls(dvc_path, detail=True):
dvc_infos[dvc_fs.path.name(info["name"])] = info
dvc_infos[dvc_fs.name(info["name"])] = info
dvc_exists = True

fs_exists = False
Expand All @@ -349,7 +383,7 @@ def ls(self, path, detail=True, dvc_only=False, **kwargs):
for info in repo.dvcignore.ls(
fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos
):
fs_infos[fs.path.name(info["name"])] = info
fs_infos[fs.name(info["name"])] = info
fs_exists = True
except (FileNotFoundError, NotADirectoryError):
pass
Expand All @@ -368,7 +402,7 @@ def ls(self, path, detail=True, dvc_only=False, **kwargs):
if not dvcfiles and _is_dvc_file(name):
continue

entry_path = self.path.join(path, name)
entry_path = self.join(path, name)
info = _merge_info(
repo, (*subkey, name), fs_infos.get(name), dvc_infos.get(name)
)
Expand Down Expand Up @@ -416,7 +450,7 @@ def _info( # noqa: C901, PLR0912
# NOTE: if some parent in fs_path turns out to be a file, it means
# that the whole repofs branch doesn't exist.
if dvc_info and not fs_info:
for parent in fs.path.parents(fs_path):
for parent in fs.parents(fs_path):
try:
if fs.info(parent)["type"] != "directory":
dvc_info = None
Expand Down Expand Up @@ -498,17 +532,16 @@ def _prepare_credentials(self, **config) -> Dict[str, Any]:
def fs(self) -> "_DVCFileSystem":
return _DVCFileSystem(**self.fs_args)

def getcwd(self):
return self.fs.getcwd()

@property
def fsid(self) -> str:
return self.fs.fsid

def isdvc(self, path, **kwargs) -> bool:
return self.fs.isdvc(path, **kwargs)

@property
def path(self) -> Path:
return self.fs.path

@property
def repo(self) -> "Repo":
return self.fs.repo
Expand Down
8 changes: 5 additions & 3 deletions dvc/fs/git.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,11 @@ def fs(

return FsspecGitFileSystem(**self.fs_args)

@functools.cached_property
def path(self):
return self.fs.path
def getcwd(self):
return self.fs.getcwd()

def chdir(self, path):
self.fs.chdir(path)

@property
def rev(self) -> str:
Expand Down
Loading