Skip to content

Commit

Permalink
Add optional file-based listings caching
Browse files Browse the repository at this point in the history
  • Loading branch information
ap-- authored and gutzbenj committed May 10, 2024
1 parent da77548 commit 5b91548
Show file tree
Hide file tree
Showing 16 changed files with 316 additions and 181 deletions.
12 changes: 10 additions & 2 deletions docs/source/api.rst
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ Base Classes
fsspec.core.OpenFiles
fsspec.core.get_fs_token_paths
fsspec.core.url_to_fs
fsspec.dircache.DirCache
fsspec.dircache.DisabledListingsCache
fsspec.dircache.MemoryListingsCache
fsspec.dircache.FileListingsCache
fsspec.FSMap
fsspec.generic.GenericFileSystem
fsspec.registry.register_implementation
Expand Down Expand Up @@ -82,7 +84,13 @@ Base Classes

.. autofunction:: fsspec.core.url_to_fs

.. autoclass:: fsspec.dircache.DirCache
.. autoclass:: fsspec.dircache.DisabledListingsCache
:members: __init__

.. autoclass:: fsspec.dircache.MemoryListingsCache
:members: __init__

.. autoclass:: fsspec.dircache.FileListingsCache
:members: __init__

.. autoclass:: fsspec.FSMap
Expand Down
8 changes: 8 additions & 0 deletions docs/source/changelog.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
Changelog
=========

Dev
--------

Enhancements

- add file-based listing cache using diskcache (#895)
warning: use new ``listings_cache_options`` instead of ``use_listings_cache`` etc.

2024.3.1
--------

Expand Down
29 changes: 20 additions & 9 deletions docs/source/features.rst
Original file line number Diff line number Diff line change
Expand Up @@ -181,15 +181,26 @@ Listings Caching
----------------

For some implementations, getting file listings (i.e., ``ls`` and anything that
depends on it) is expensive. These implementations use dict-like instances of
:class:`fsspec.dircache.DirCache` to manage the listings.

The cache allows for time-based expiry of entries with the ``listings_expiry_time``
parameter, or LRU expiry with the ``max_paths`` parameter. These can be
set on any implementation instance that uses listings caching; or to skip the
caching altogether, use ``use_listings_cache=False``. That would be appropriate
when the target location is known to be volatile because it is being written
to from other sources.
depends on it) is expensive. These implementations maye use either dict-like instances of
:class:`fsspec.dircache.MemoryListingsCache` or file-based caching with instances of
:class:`fsspec.dircache.FileListingsCache` to manage the listings.

The listings cache can be controlled via the keyword ``listings_cache_options`` which is a dictionary.
The type of cache that is used, can be controlled via the keyword ``cache_type`` (`disabled`, `memory` or `file`).
The cache allows for time-based expiry of entries with the keyword ``expiry_time``. If the target location is known to
be volatile because e.g. it is being written to from other sources we recommend to disable the listings cache.
If you want to use the file-based caching, you can also provide the argument
``directory`` to determine where the cache file is stored.

Example for ``listings_cache_options``:

.. code-block:: json
{
"cache_type": "file",
"expiry_time": 3600,
"directory": "/tmp/cache"
}
When the ``fsspec`` instance writes to the backend, the method ``invalidate_cache``
is called, so that subsequent listing of the given paths will force a refresh. In
Expand Down
12 changes: 6 additions & 6 deletions fsspec/archive.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,19 +37,19 @@ def _all_dirnames(self, paths):
def info(self, path, **kwargs):
self._get_dirs()
path = self._strip_protocol(path)
if path in {"", "/"} and self.dir_cache:
if path in {"", "/"} and self.dircache:
return {"name": "", "type": "directory", "size": 0}
if path in self.dir_cache:
return self.dir_cache[path]
elif path + "/" in self.dir_cache:
return self.dir_cache[path + "/"]
if path in self.dircache:
return self.dircache[path]
elif path + "/" in self.dircache:
return self.dircache[path + "/"]
else:
raise FileNotFoundError(path)

def ls(self, path, detail=True, **kwargs):
self._get_dirs()
paths = {}
for p, f in self.dir_cache.items():
for p, f in self.dircache.items():
p = p.rstrip("/")
if "/" in p:
root = p.rsplit("/", 1)[0]
Expand Down
12 changes: 10 additions & 2 deletions fsspec/asyn.py
Original file line number Diff line number Diff line change
Expand Up @@ -312,15 +312,23 @@ class AsyncFileSystem(AbstractFileSystem):
mirror_sync_methods = True
disable_throttling = False

def __init__(self, *args, asynchronous=False, loop=None, batch_size=None, **kwargs):
def __init__(
self,
*args,
asynchronous=False,
loop=None,
batch_size=None,
listings_cache_options=None,
**kwargs,
):
self.asynchronous = asynchronous
self._pid = os.getpid()
if not asynchronous:
self._loop = loop or get_loop()
else:
self._loop = None
self.batch_size = batch_size
super().__init__(*args, **kwargs)
super().__init__(listings_cache_options, *args, **kwargs)

@property
def loop(self):
Expand Down
98 changes: 0 additions & 98 deletions fsspec/dircache.py

This file was deleted.

47 changes: 33 additions & 14 deletions fsspec/implementations/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import logging
import re
import weakref
from copy import copy
from urllib.parse import urlparse

import aiohttp
Expand Down Expand Up @@ -58,6 +57,7 @@ def __init__(
client_kwargs=None,
get_client=get_client,
encoded=False,
listings_cache_options=None,
**storage_options,
):
"""
Expand All @@ -83,11 +83,39 @@ def __init__(
A callable which takes keyword arguments and constructs
an aiohttp.ClientSession. It's state will be managed by
the HTTPFileSystem class.
listings_cache_options: dict
Options for the listings cache.
storage_options: key-value
Any other parameters passed on to requests
cache_type, cache_options: defaults used in open
"""
super().__init__(self, asynchronous=asynchronous, loop=loop, **storage_options)
# TODO: remove in future release
# Clean caching-related parameters from `storage_options`
# before propagating them as `request_options` through `self.kwargs`.
old_listings_cache_kwargs = {
"use_listings_cache",
"listings_expiry_time",
"max_paths",
"skip_instance_cache",
}
# intersection of old_listings_cache_kwargs and storage_options
old_listings_cache_kwargs = old_listings_cache_kwargs.intersection(
storage_options
)
if old_listings_cache_kwargs:
logger.warning(
f"The following parameters are not used anymore and will be ignored: {old_listings_cache_kwargs}. "
f"Use new `listings_cache_options` instead."
)
for key in old_listings_cache_kwargs:
del storage_options[key]
super().__init__(
self,
asynchronous=asynchronous,
loop=loop,
listings_cache_options=listings_cache_options,
**storage_options,
)
self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE
self.simple_links = simple_links
self.same_schema = same_scheme
Expand All @@ -96,19 +124,10 @@ def __init__(
self.client_kwargs = client_kwargs or {}
self.get_client = get_client
self.encoded = encoded
self.kwargs = storage_options
self._session = None

# Clean caching-related parameters from `storage_options`
# before propagating them as `request_options` through `self.kwargs`.
# TODO: Maybe rename `self.kwargs` to `self.request_options` to make
# it clearer.
request_options = copy(storage_options)
self.use_listings_cache = request_options.pop("use_listings_cache", False)
request_options.pop("listings_expiry_time", None)
request_options.pop("max_paths", None)
request_options.pop("skip_instance_cache", None)
self.kwargs = request_options
self.kwargs = storage_options
self._session = None

@property
def fsid(self):
Expand Down Expand Up @@ -201,7 +220,7 @@ async def _ls_real(self, url, detail=True, **kwargs):
return sorted(out)

async def _ls(self, url, detail=True, **kwargs):
if self.use_listings_cache and url in self.dircache:
if url in self.dircache:
out = self.dircache[url]
else:
out = await self._ls_real(url, detail=detail, **kwargs)
Expand Down
2 changes: 1 addition & 1 deletion fsspec/implementations/libarchive.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ def __init__(
Kwargs passed when instantiating the target FS, if ``fo`` is
a string.
"""
super().__init__(self, **kwargs)
super().__init__(False, self, **kwargs)
if mode != "r":
raise ValueError("Only read from archive files accepted")
if isinstance(fo, str):
Expand Down
8 changes: 4 additions & 4 deletions fsspec/implementations/tar.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def __init__(
self._fo_ref = fo
self.fo = fo # the whole instance is a context
self.tar = tarfile.TarFile(fileobj=self.fo)
self.dir_cache = None
self.dircache = None

self.index_store = index_store
self.index = None
Expand All @@ -101,19 +101,19 @@ def _index(self):
# TODO: save index to self.index_store here, if set

def _get_dirs(self):
if self.dir_cache is not None:
if self.dircache is not None:
return

# This enables ls to get directories as children as well as files
self.dir_cache = {
self.dircache = {
dirname: {"name": dirname, "size": 0, "type": "directory"}
for dirname in self._all_dirnames(self.tar.getnames())
}
for member in self.tar.getmembers():
info = member.get_info()
info["name"] = info["name"].rstrip("/")
info["type"] = typemap.get(info["type"], "file")
self.dir_cache[info["name"]] = info
self.dircache[info["name"]] = info

def _open(self, path, mode="rb", **kwargs):
if mode != "rb":
Expand Down
Loading

0 comments on commit 5b91548

Please sign in to comment.