Skip to content

Commit

Permalink
fix: correctly handle local files with colons (:) in name (#1452)
Browse files Browse the repository at this point in the history
---------

Co-authored-by: Martin Durant <martindurant@users.noreply.github.com>
  • Loading branch information
lobis and martindurant committed Dec 7, 2023
1 parent 30e479b commit 7c3408a
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 4 deletions.
2 changes: 1 addition & 1 deletion fsspec/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,7 @@ def split_protocol(urlpath):
if len(protocol) > 1:
# excludes Windows paths
return protocol, path
if ":" in urlpath and urlpath.find(":") > 1:
if urlpath.startswith("data:"):
return urlpath.split(":", 1)
return None, urlpath

Expand Down
7 changes: 5 additions & 2 deletions fsspec/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,9 +250,12 @@ async def _pipe_file(
return fs.pipe_file(path, value, **kwargs)

async def _rm(self, url, **kwargs):
fs = _resolve_fs(url, self.method)
urls = url
if isinstance(urls, str):
urls = [urls]
fs = _resolve_fs(urls[0], self.method)
if fs.async_impl:
await fs._rm(url, **kwargs)
await fs._rm(urls, **kwargs)
else:
fs.rm(url, **kwargs)

Expand Down
21 changes: 20 additions & 1 deletion fsspec/implementations/tests/test_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
),
}


csv_files = {
".test.fakedata.1.csv": (b"a,b\n" b"1,2\n"),
".test.fakedata.2.csv": (b"a,b\n" b"3,4\n"),
Expand All @@ -56,6 +55,8 @@ def filetexts(d, open=open, mode="t"):
try:
os.chdir(dirname)
for filename, text in d.items():
if dirname := os.path.dirname(filename):
os.makedirs(dirname, exist_ok=True)
f = open(filename, f"w{mode}")
try:
f.write(text)
Expand Down Expand Up @@ -991,3 +992,21 @@ def test_cp_two_files(tmpdir):
make_path_posix(os.path.join(target, "file0")),
make_path_posix(os.path.join(target, "file1")),
]


@pytest.mark.skipif(WIN, reason="Windows does not support colons in filenames")
def test_issue_1447():
files_with_colons = {
".local:file:with:colons.txt": b"content1",
".colons-after-extension.txt:after": b"content2",
".colons-after-extension/file:colon.txt:before/after": b"content3",
}
with filetexts(files_with_colons, mode="b"):
for file, contents in files_with_colons.items():
with fsspec.filesystem("file").open(file, "rb") as f:
assert f.read() == contents

fs, urlpath = fsspec.core.url_to_fs(file)
assert isinstance(fs, fsspec.implementations.local.LocalFileSystem)
with fs.open(urlpath, "rb") as f:
assert f.read() == contents

0 comments on commit 7c3408a

Please sign in to comment.