ls: support listing versions (#661)

* deps: bump moto to >= 4

* ls: support listing versions

.github/workflows/ci.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Install dependencies
         shell: bash -l {0}
         run: |
-          conda install -c conda-forge pip botocore aiobotocore==2.4.0 "aiohttp" "werkzeug<2.2.0" "moto>=2.0,<3" pytest flake8 black -y
+          conda install -c conda-forge pip botocore aiobotocore==2.4.0 "aiohttp" "werkzeug<2.2.0" "moto>=4" pytest flake8 black -y
           pip install git+https://github.com/fsspec/filesystem_spec --no-deps
           conda list
           conda --version

s3fs/core.py

@@ -652,14 +652,25 @@ def _open(
         )
 
     async def _lsdir(
-        self, path, refresh=False, max_items=None, delimiter="/", prefix=""
+        self,
+        path,
+        refresh=False,
+        max_items=None,
+        delimiter="/",
+        prefix="",
+        versions=False,
     ):
+        if versions and not self.version_aware:
+            raise ValueError(
+                "versions cannot be specified if the filesystem is not version aware"
+            )
+
         bucket, key, _ = self.split_path(path)
         if not prefix:
             prefix = ""
         if key:
             prefix = key.lstrip("/") + "/" + prefix
-        if path not in self.dircache or refresh or not delimiter:
+        if path not in self.dircache or refresh or not delimiter or versions:
             try:
                 logger.debug("Get directory listing page for %s" % path)
                 await self.set_session()
@@ -686,7 +697,7 @@ async def _lsdir(
                 async for i in it:
                     dircache.extend(i.get("CommonPrefixes", []))
                     for c in i.get(contents_key, []):
-                        if not self.version_aware or c.get("IsLatest"):
+                        if not self.version_aware or c.get("IsLatest") or versions:
                             c["type"] = "file"
                             c["size"] = c["Size"]
                             files.append(c)
@@ -706,10 +717,13 @@ async def _lsdir(
                 for f in files:
                     f["Key"] = "/".join([bucket, f["Key"]])
                     f["name"] = f["Key"]
+                    version_id = f.get("VersionId")
+                    if versions and version_id and version_id != "null":
+                        f["name"] += f"?versionId={version_id}"
             except ClientError as e:
                 raise translate_boto_error(e)
 
-            if delimiter and files:
+            if delimiter and files and not versions:
                 self.dircache[path] = files
             return files
         return self.dircache[path]
@@ -882,7 +896,7 @@ async def _lsbuckets(self, refresh=False):
             return files
         return self.dircache[""]
 
-    async def _ls(self, path, detail=False, refresh=False):
+    async def _ls(self, path, detail=False, refresh=False, versions=False):
         """List files in given bucket, or list of buckets.
 
         Listing is cached unless `refresh=True`.
@@ -901,9 +915,11 @@ async def _ls(self, path, detail=False, refresh=False):
         if path in ["", "/"]:
             files = await self._lsbuckets(refresh)
         else:
-            files = await self._lsdir(path, refresh)
+            files = await self._lsdir(path, refresh, versions=versions)
             if not files and "/" in path:
-                files = await self._lsdir(self._parent(path), refresh=refresh)
+                files = await self._lsdir(
+                    self._parent(path), refresh=refresh, versions=versions
+                )
                 files = [
                     o
                     for o in files

s3fs/tests/test_s3fs.py

@@ -1602,6 +1602,12 @@ def test_versioned_file_fullpath(s3):
         assert fo.version_id == version_id
         assert fo.read() == b"1"
 
+    versions = s3.object_version_info(versioned_file)
+    version_ids = [version["VersionId"] for version in versions]
+    assert set(s3.ls(versioned_bucket_name, versions=True)) == {
+        f"{versioned_file}?versionId={vid}" for vid in version_ids
+    }
+
 
 def test_versions_unaware(s3):
     versioned_file = versioned_bucket_name + "/versioned_file3"

test_requirements.txt

@@ -1,5 +1,5 @@
 mock; python_version < '3.3'
-moto>=2,<3
+moto>=4
 flask
 pytest>=4.2.0
 pytest-env