diff --git a/doit/cmd_base.py b/doit/cmd_base.py index 2b62e241..7076ef95 100644 --- a/doit/cmd_base.py +++ b/doit/cmd_base.py @@ -5,7 +5,7 @@ from . import version from .cmdparse import CmdOption, CmdParse from .exceptions import InvalidCommand, InvalidDodoFile -from .dependency import CHECKERS, DbmDB, JsonDB, SqliteDB, Dependency +from .dependency import CHECKERS, DbmDB, JsonDB, SqliteDB, Dependency, RedisDB from .plugin import PluginDict from . import loader @@ -366,7 +366,8 @@ def _get_loader(self, task_loader=None, cmds=None): def get_backends(self): """return PluginDict of DB backends, including core and plugins""" - backend_map = {'dbm': DbmDB, 'json': JsonDB, 'sqlite3': SqliteDB} + backend_map = {'dbm': DbmDB, 'json': JsonDB, 'sqlite3': SqliteDB, + 'redis': RedisDB} # add plugins plugins = PluginDict() plugins.add_plugins(self.config, 'BACKEND') diff --git a/doit/dependency.py b/doit/dependency.py index 84973144..5e858c00 100644 --- a/doit/dependency.py +++ b/doit/dependency.py @@ -227,6 +227,66 @@ def remove_all(self): self.dirty = set() +class RedisDB(object): + """Backend using Redis. + + Parameters to open the database can be passed with the url format:: + + redis://[:password]@localhost:6379/0 + + """ + def __init__(self, name): + import redis + self.name = name + self._dbm = redis.from_url(name) + self._db = defaultdict(dict) + self.dirty = set() + + def dump(self): + """save/close DBM file""" + for task_id in self.dirty: + self._dbm[task_id] = json.dumps(self._db[task_id]) + self.dirty = set() + + sync = dump + + def set(self, task_id, dependency, value): + """Store value in the DB.""" + self._db[task_id][dependency] = value + self.dirty.add(task_id) + + def get(self, task_id, dependency): + """Get value stored in the DB.""" + # optimization, just try to get it without checking it exists + if task_id in self._db: + return self._db[task_id].get(dependency, None) + else: + try: + task_data = self._dbm[task_id] + except KeyError: + return + self._db[task_id] = json.loads(task_data.decode('utf-8')) + return self._db[task_id].get(dependency, None) + + def in_(self, task_id): + """@return bool if task_id is in DB""" + return task_id in self._dbm or task_id in self.dirty + + def remove(self, task_id): + """remove saved dependecies from DB for taskId""" + if task_id in self._db: + del self._db[task_id] + if task_id in self._dbm: + del self._dbm[task_id] + if task_id in self.dirty: + self.dirty.remove(task_id) + + def remove_all(self): + """remove saved dependecies from DB for all tasks""" + self._db = defaultdict(dict) + self._dbm.flushdb() + self.dirty = set() + class SqliteDB(object): """ sqlite3 json backend """ diff --git a/tests/conftest.py b/tests/conftest.py index 46e599a1..a048d6a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ import py import pytest -from doit.dependency import DbmDB, Dependency, MD5Checker +from doit.dependency import DbmDB, Dependency, MD5Checker, RedisDB from doit.task import Task @@ -86,13 +86,21 @@ def depfile(request): name = py.std.re.sub("[\W]", "_", name) my_tmpdir = request.config._tmpdirhandler.mktemp(name, numbered=True) dep_file = Dependency(dep_class, os.path.join(my_tmpdir.strpath, "testdb")) - dep_file.whichdb = whichdb(dep_file.name) if dep_class is DbmDB else 'XXX' + if dep_class is DbmDB: + dep_file.whichdb = whichdb(dep_file.name) + elif dep_class is RedisDB: + dep_file.whichdb = 'redis' + else: + dep_file.whichdb = 'XXX' dep_file.name_ext = db_ext.get(dep_file.whichdb, ['']) def remove_depfile(): if not dep_file._closed: dep_file.close() - remove_db(dep_file.name) + if dep_class is RedisDB: + dep_file.backend.remove_all() + else: + remove_db(dep_file.name) request.addfinalizer(remove_depfile) return dep_file diff --git a/tests/test_dependency.py b/tests/test_dependency.py index 3abcb17e..86b8d54d 100644 --- a/tests/test_dependency.py +++ b/tests/test_dependency.py @@ -9,7 +9,7 @@ from doit.task import Task from doit.dependency import get_md5, get_file_md5 -from doit.dependency import DbmDB, JsonDB, SqliteDB, Dependency +from doit.dependency import DbmDB, JsonDB, SqliteDB, Dependency, RedisDB from doit.dependency import DatabaseException, UptodateCalculator from doit.dependency import FileChangedChecker, MD5Checker, TimestampChecker from doit.dependency import DependencyStatus @@ -69,7 +69,14 @@ def test_sqlite_import(): @pytest.fixture def pdepfile(request): return depfile(request) -pytest.fixture(params=[JsonDB, DbmDB, SqliteDB])(pdepfile) + +try: + import redis + db = redis.StrictRedis() + db.client_getname() + pytest.fixture(params=[JsonDB, DbmDB, SqliteDB, RedisDB])(pdepfile) +except: + pytest.fixture(params=[JsonDB, DbmDB, SqliteDB])(pdepfile) # FIXME there was major refactor breaking classes from dependency, # unit-tests could be more specific to base classes. @@ -101,6 +108,8 @@ def test_dump(self, pdepfile): def test_corrupted_file(self, pdepfile): if pdepfile.whichdb is None: # pragma: no cover pytest.skip('dumbdbm too dumb to detect db corruption') + if pdepfile.whichdb is 'redis': # pragma: no cover + pytest.skip('redis too dumb to detect db corruption') # create some corrupted files for name_ext in pdepfile.name_ext: