Skip to content

Commit

Permalink
Split cache's activate and sync steps into 2 ctx managers
Browse files Browse the repository at this point in the history
Split the cache context manager into two: one for activate the cache
(patch on enter and unpatch on exit) and the other to sync the cache to
disk (clear the cache's stats on enter and sync if needed to disk on
exit).

This simplified the code and enabled to open and load the cache just
once, avoiding if possible any hit to the disk.
  • Loading branch information
eldipa committed Nov 13, 2018
1 parent 631108f commit 99ff4d1
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 17 deletions.
16 changes: 9 additions & 7 deletions byexample/byexample.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
import os, sys

def execute_examples(filename, sigint_handler):
global cache_disabled, cache_verbose, harvester, executor, options, human_args, dry
global cache, harvester, executor, options, human_args, dry
from .common import human_exceptions

with RegexCache(filename, cache_disabled, cache_verbose), \
human_exceptions("File '%s':" % filename, *human_args) as exc, \
with human_exceptions("File '%s':" % filename, *human_args) as exc, \
cache.synced(label=filename), \
allow_sigint(sigint_handler):
examples = harvester.get_examples_from_file(filename)
if dry:
Expand All @@ -21,11 +21,13 @@ def execute_examples(filename, sigint_handler):
return True, True, user_aborted, error

def main(args=None):
global cache_disabled, cache_verbose, harvester, executor, options, human_args, dry
global cache, harvester, executor, options, human_args, dry

cache_disabled = os.getenv('BYEXAMPLE_CACHE_DISABLED', "0") != "0"
cache_verbose = os.getenv('BYEXAMPLE_CACHE_VERBOSE', "0") != "0"
with RegexCache('0', cache_disabled, cache_verbose):
cache = RegexCache('0', cache_disabled, cache_verbose)

with cache.activated(auto_sync=True, label="0"):
from .cmdline import parse_args
from .common import human_exceptions
from .init import init
Expand All @@ -39,5 +41,5 @@ def main(args=None):
if exc:
sys.exit(Status.error)

jobs = Jobs(args.jobs, args.verbosity)
return jobs.run(execute_examples, testfiles, options['fail_fast'])
jobs = Jobs(args.jobs, args.verbosity)
return jobs.run(execute_examples, testfiles, options['fail_fast'])
50 changes: 40 additions & 10 deletions byexample/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
except NameError:
unicode = str


@contextlib.contextmanager
def flock(file, shared=False):
fcntl.lockf(file.fileno(), fcntl.LOCK_SH if shared else fcntl.LOCK_EX)
Expand All @@ -37,9 +36,41 @@ def __init__(self, filename, disabled=False, cache_verbose=False):
self.verbose = cache_verbose

self._cache = self._load_cache_from_disk()
self._nkeys, self._hits = len(self._cache), 0
self.clear_stats()
self._log("Cache '%s': %i entries" % (self.filename, self._nkeys))

@contextlib.contextmanager
def synced(self, label=""):
''' Clear the cache's stats on enter and sync the cache
on exit.
'''
self.clear_stats()
try:
yield self
finally:
self.sync(label)

@contextlib.contextmanager
def activated(self, auto_sync, label=""):
''' Activate the cache (patch re.compile) on enter
and deactivate it on exit.
If auto_sync is True, also clear the cache's stats
on enter and sync the cache on exit.
'''
self._patch()
try:
if auto_sync:
with self.synced(label):
yield self
else:
yield self
finally:
self._unpatch()

def clear_stats(self):
self._nkeys, self._hits = len(self._cache), 0

@classmethod
def _cache_filepath(cls, filename):
''' Create a valid file path based on <filename>.
Expand Down Expand Up @@ -122,14 +153,14 @@ def _create_empty_cache_in_disk(self):

return cache

def sync(self):
def sync(self, label=""):
misses = len(self._cache) - self._nkeys
nohits = self._nkeys - self._hits

self._log("Cache '%s' stats: %i entries %i hits %i misses %i nohits." \
% (self.filename, len(self._cache), self._hits, misses, nohits))
self._log("[%s] Cache stats: %i entries %i hits %i misses %i nohits." \
% (label, len(self._cache), self._hits, misses, nohits))
if misses and not self.disabled and self.filename != None:
self._log("Cache '%s' require sync." % self.filename)
self._log("[%s] Cache require sync." % label)
with open(self.filename, 'rb+') as f, flock(f):
# get a fresh disk version in case that other
# byexample instance had touched the cache but
Expand All @@ -147,7 +178,7 @@ def sync(self):
pickle.dump(cache, f)
f.truncate()

self._nkeys, self._hits = len(self._cache), 0
self.clear_stats()

def get(self, pattern, flags=0):
''' RegexCache.get compiles a pattern into a regex object like
Expand Down Expand Up @@ -221,7 +252,7 @@ def _bytecode_to_regex(self, pattern, bytecode):
groupindex, indexgroup
)

def __enter__(self):
def _patch(self):
if self.disabled:
return self

Expand All @@ -231,10 +262,9 @@ def __enter__(self):

return self

def __exit__(self, *args, **kargs):
def _unpatch(self, *args, **kargs):
if self.disabled:
return

sre_compile.compile = self._original__sre_compile__compile
self.sync()

0 comments on commit 99ff4d1

Please sign in to comment.