fs: temporarily disable prefix-based search for fsspec-implementations

dvc/fs/fsspec_wrapper.py

@@ -11,6 +11,8 @@
 
 # pylint: disable=no-member
 class FSSpecWrapper(BaseFileSystem):
+    TRAVERSE_PREFIX_LEN = 2
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.fs_args = {"skip_instance_cache": True}

dvc/objects/db/base.py

@@ -399,8 +399,15 @@ def hashes_exist(self, hashes, jobs=None, name=None):
         # hashes_exist() (see ssh, local)
         assert self.fs.TRAVERSE_PREFIX_LEN >= 2
 
+        # During the tests, for ensuring that the traverse behavior
+        # is working we turn on this option. It will ensure the
+        # list_hashes_traverse() is called.
+        always_traverse = getattr(self.fs, "_ALWAYS_TRAVERSE", False)
+
         hashes = set(hashes)
-        if len(hashes) == 1 or not self.fs.CAN_TRAVERSE:
+        if (
+            len(hashes) == 1 or not self.fs.CAN_TRAVERSE
+        ) and not always_traverse:
             remote_hashes = self.list_hashes_exists(hashes, jobs, name)
             return remote_hashes
 
@@ -418,7 +425,7 @@ def hashes_exist(self, hashes, jobs=None, name=None):
             )
         else:
             traverse_weight = traverse_pages
-        if len(hashes) < traverse_weight:
+        if len(hashes) < traverse_weight and not always_traverse:
             logger.debug(
                 "Large remote ('{}' hashes < '{}' traverse weight), "
                 "using object_exists for remaining hashes".format(

tests/func/test_data_cloud.py

@@ -41,6 +41,14 @@
     )
 ]
 
+# Clouds that implement the general methods that can be tested
+# for functional tests that require extensive apis (e.g traversing
+# via walk_files())
+full_clouds = [
+    pytest.lazy_fixture(cloud)
+    for cloud in ["s3", "gs", "azure", "ssh", "hdfs"]
+]
+
 
 @pytest.mark.needs_internet
 @pytest.mark.parametrize("remote", all_clouds, indirect=True)
@@ -548,3 +556,18 @@ def test_pull_partial(tmp_dir, dvc, local_remote):
     stats = dvc.pull(os.path.join("foo", "bar"))
     assert stats["fetched"] == 1
     assert (tmp_dir / "foo").read_text() == {"bar": {"baz": "baz"}}
+
+
+@pytest.mark.parametrize("remote", full_clouds, indirect=True)
+def test_pull_00_prefix(tmp_dir, dvc, remote, monkeypatch):
+    fs_type = type(dvc.cloud.get_remote("upstream").fs)
+    monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
+    monkeypatch.setattr(fs_type, "LIST_OBJECT_PAGE_SIZE", 256, raising=False)
+
+    tmp_dir.dvc_gen({"foo": "363"})
+
+    dvc.push()
+    clean(["foo"], dvc)
+
+    stats = dvc.pull()
+    assert "foo" in stats["added"]