Skip to content

Add time based update for caching in fs.wrap.WrapCachedDir #140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
media-proxy wants to merge 11 commits into from
Closed

Add time based update for caching in fs.wrap.WrapCachedDir #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
9525029
Add time based update for caching in fs.wrap.WrapCachedDir
merlink01 Feb 10, 2018
0364b59
Fix Import problem
merlink01 Feb 11, 2018
ffac125
python2 doesnt like from time import time?
merlink01 Feb 11, 2018
ef65826
time seems to be overwritten somewhere, try renaming it
merlink01 Feb 11, 2018
54aff1e
Strange error with time???
merlink01 Feb 11, 2018
899c7ce
Strange error with time???
merlink01 Feb 11, 2018
b9c67f7
Problem seems to fs.time module...
merlink01 Feb 11, 2018
0543a67
Try absolute import
merlink01 Feb 11, 2018
38a189a
Add some tests
merlink01 Feb 11, 2018
3fd95a6
fix
merlink01 Feb 11, 2018
c1157ff
tests are running now
merlink01 Feb 11, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 25 additions & 7 deletions fs/wrap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@

from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import

from time import time as gettime
from .wrapfs import WrapFS
from .path import abspath, normpath, split
from .errors import ResourceReadOnly, ResourceNotFound
Expand All @@ -36,7 +38,7 @@ def read_only(fs):
return WrapReadOnly(fs)


def cache_directory(fs):
def cache_directory(fs,livetime=-1,speedup=False):
"""Make a filesystem that caches directory information.

Arguments:
Expand All @@ -47,7 +49,7 @@ def cache_directory(fs):
and other methods which read directory information.

"""
return WrapCachedDir(fs)
return WrapCachedDir(fs, livetime, speedup)


class WrapCachedDir(WrapFS):
Expand All @@ -67,8 +69,10 @@ class WrapCachedDir(WrapFS):

wrap_name = 'cached-dir'

def __init__(self, wrap_fs):
def __init__(self, wrap_fs, livetime=10, speedup=False):
super(WrapCachedDir, self).__init__(wrap_fs)
self.livetime = livetime
self.speedup = speedup
self._cache = {}

def scandir(self, path, namespaces=None, page=None):
Expand All @@ -81,7 +85,21 @@ def scandir(self, path, namespaces=None, page=None):
page=page
)
_dir = {info.name: info for info in _scan_result}
self._cache[cache_key] = _dir
self._cache[cache_key] = {'time':gettime(),'data':_dir}
else:
if self.livetime >= 0:
if self._cache[cache_key]['time'] + self.livetime < gettime():
_scan_result = self._wrap_fs.scandir(
path,
namespaces=namespaces,
page=page
)
_dir = {info.name: info for info in _scan_result}
self._cache[cache_key] = {'time':gettime(),'data':_dir}
else:
if self.speedup:
self._cache[cache_key]['time'] = gettime()

gen_scandir = iter(self._cache[cache_key].values())
return gen_scandir

Expand All @@ -97,12 +115,12 @@ def getinfo(self, path, *namespaces):
dir_path, resource_name = split(_path)
cache_key = (dir_path, frozenset(namespaces or ()))

if cache_key not in self._cache:
self.scandir(dir_path, namespaces=namespaces)
#if cache_key not in self._cache:
self.scandir(dir_path, namespaces=namespaces)

_dir = self._cache[cache_key]
try:
info = _dir[resource_name]
info = _dir['data'][resource_name]
except KeyError:
raise ResourceNotFound(path)
return info
Expand Down
62 changes: 61 additions & 1 deletion tests/test_wrap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import unittest

from time import sleep
from fs import errors
from fs import open_fs
from fs import wrap
Expand Down Expand Up @@ -80,7 +81,7 @@ def test_cachedir(self):
mem_fs.makedirs('foo/bar/baz')
mem_fs.touch('egg')

fs = wrap.cache_directory(mem_fs)
fs = wrap.cache_directory(mem_fs,livetime=-1)
self.assertEqual(
sorted(fs.listdir('/')),
['egg', 'foo']
Expand All @@ -103,3 +104,62 @@ def test_cachedir(self):
with self.assertRaises(errors.ResourceNotFound):
fs.getinfo('/foofoo')

def test_cachedir_time(self):
Copy link
Member

@willmcgugan willmcgugan Mar 10, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you familiar with https://github.com/spulec/freezegun ? You can get rid of those sleep in tests.

mem_fs = open_fs('mem://')
mem_fs.makedirs('foo/bar/baz')
mem_fs.touch('egg')

fs = wrap.cache_directory(mem_fs,livetime=3)
self.assertEqual(
sorted(fs.listdir('/')),
['egg', 'foo']
)
self.assertEqual(
sorted(fs.listdir('/')),
['egg', 'foo']
)

#caching dir
self.assertTrue(fs.isdir('foo/bar/baz'))
mem_fs.removedir('foo/bar/baz')
mem_fs.touch('foo/bar/baz')
self.assertTrue(fs.isdir('foo/bar/baz'))
sleep(0.5)
self.assertTrue(fs.isdir('foo/bar/baz'))
sleep(0.5)
self.assertTrue(fs.isdir('foo/bar/baz'))
sleep(3)
self.assertFalse(fs.isdir('foo/bar/baz'))

def test_cachedir_speedup(self):
mem_fs = open_fs('mem://')
mem_fs.makedirs('foo/bar/baz')
mem_fs.touch('egg')

fs = wrap.cache_directory(mem_fs,livetime=3,speedup=True)
self.assertEqual(
sorted(fs.listdir('/')),
['egg', 'foo']
)
self.assertEqual(
sorted(fs.listdir('/')),
['egg', 'foo']
)

#caching dir
self.assertTrue(fs.isdir('foo/bar/baz'))
mem_fs.removedir('foo/bar/baz')
mem_fs.touch('foo/bar/baz')
self.assertTrue(fs.isdir('foo/bar/baz'))
sleep(2)
self.assertTrue(fs.isdir('foo/bar/baz'))
sleep(2)
self.assertTrue(fs.isdir('foo/bar/baz'))
sleep(2)
self.assertTrue(fs.isdir('foo/bar/baz'))
sleep(4)
self.assertFalse(fs.isdir('foo/bar/baz'))