Add a Redis backend

doit/cmd_base.py

@@ -5,7 +5,7 @@
 from . import version
 from .cmdparse import CmdOption, CmdParse
 from .exceptions import InvalidCommand, InvalidDodoFile
-from .dependency import CHECKERS, DbmDB, JsonDB, SqliteDB, Dependency
+from .dependency import CHECKERS, DbmDB, JsonDB, SqliteDB, Dependency, RedisDB
 from .plugin import PluginDict
 from . import loader
 
@@ -366,7 +366,8 @@ def _get_loader(self, task_loader=None, cmds=None):
 
     def get_backends(self):
         """return PluginDict of DB backends, including core and plugins"""
-        backend_map = {'dbm': DbmDB, 'json': JsonDB, 'sqlite3': SqliteDB}
+        backend_map = {'dbm': DbmDB, 'json': JsonDB, 'sqlite3': SqliteDB,
+                       'redis': RedisDB}
         # add plugins
         plugins = PluginDict()
         plugins.add_plugins(self.config, 'BACKEND')

doit/dependency.py

@@ -227,6 +227,66 @@ def remove_all(self):
         self.dirty = set()
 
 
+class RedisDB(object):
+    """Backend using Redis.
+
+    Parameters to open the database can be passed with the url format::
+
+        redis://[:password]@localhost:6379/0
+
+    """
+    def __init__(self, name):
+        import redis
+        self.name = name
+        self._dbm = redis.from_url(name)
+        self._db = defaultdict(dict)
+        self.dirty = set()
+
+    def dump(self):
+        """save/close DBM file"""
+        for task_id in self.dirty:
+            self._dbm[task_id] = json.dumps(self._db[task_id])
+        self.dirty = set()
+
+    sync = dump
+
+    def set(self, task_id, dependency, value):
+        """Store value in the DB."""
+        self._db[task_id][dependency] = value
+        self.dirty.add(task_id)
+
+    def get(self, task_id, dependency):
+        """Get value stored in the DB."""
+        # optimization, just try to get it without checking it exists
+        if task_id in self._db:
+            return self._db[task_id].get(dependency, None)
+        else:
+            try:
+                task_data = self._dbm[task_id]
+            except KeyError:
+                return
+            self._db[task_id] = json.loads(task_data.decode('utf-8'))
+            return self._db[task_id].get(dependency, None)
+
+    def in_(self, task_id):
+        """@return bool if task_id is in DB"""
+        return task_id in self._dbm or task_id in self.dirty
+
+    def remove(self, task_id):
+        """remove saved dependecies from DB for taskId"""
+        if task_id in self._db:
+            del self._db[task_id]
+        if task_id in self._dbm:
+            del self._dbm[task_id]
+        if task_id in self.dirty:
+            self.dirty.remove(task_id)
+
+    def remove_all(self):
+        """remove saved dependecies from DB for all tasks"""
+        self._db = defaultdict(dict)
+        self._dbm.flushdb()
+        self.dirty = set()
+
 
 class SqliteDB(object):
     """ sqlite3 json backend """

tests/conftest.py

@@ -5,7 +5,7 @@
 import py
 import pytest
 
-from doit.dependency import DbmDB, Dependency, MD5Checker
+from doit.dependency import DbmDB, Dependency, MD5Checker, RedisDB
 from doit.task import Task
 
 
@@ -86,13 +86,21 @@ def depfile(request):
     name = py.std.re.sub("[\W]", "_", name)
     my_tmpdir = request.config._tmpdirhandler.mktemp(name, numbered=True)
     dep_file = Dependency(dep_class, os.path.join(my_tmpdir.strpath, "testdb"))
-    dep_file.whichdb = whichdb(dep_file.name) if dep_class is DbmDB else 'XXX'
+    if dep_class is DbmDB:
+        dep_file.whichdb = whichdb(dep_file.name)
+    elif dep_class is RedisDB:
+        dep_file.whichdb = 'redis'
+    else:
+        dep_file.whichdb = 'XXX'
     dep_file.name_ext = db_ext.get(dep_file.whichdb, [''])
 
     def remove_depfile():
         if not dep_file._closed:
             dep_file.close()
-        remove_db(dep_file.name)
+        if dep_class is RedisDB:
+            dep_file.backend.remove_all()
+        else:
+            remove_db(dep_file.name)
     request.addfinalizer(remove_depfile)
 
     return dep_file

tests/test_dependency.py

@@ -9,7 +9,7 @@
 
 from doit.task import Task
 from doit.dependency import get_md5, get_file_md5
-from doit.dependency import DbmDB, JsonDB, SqliteDB, Dependency
+from doit.dependency import DbmDB, JsonDB, SqliteDB, Dependency, RedisDB
 from doit.dependency import DatabaseException, UptodateCalculator
 from doit.dependency import FileChangedChecker, MD5Checker, TimestampChecker
 from doit.dependency import DependencyStatus
@@ -69,7 +69,14 @@ def test_sqlite_import():
 @pytest.fixture
 def pdepfile(request):
     return depfile(request)
-pytest.fixture(params=[JsonDB, DbmDB, SqliteDB])(pdepfile)
+
+try:
+    import redis
+    db = redis.StrictRedis()
+    db.client_getname()
+    pytest.fixture(params=[JsonDB, DbmDB, SqliteDB, RedisDB])(pdepfile)
+except:
+    pytest.fixture(params=[JsonDB, DbmDB, SqliteDB])(pdepfile)
 
 # FIXME there was major refactor breaking classes from dependency,
 # unit-tests could be more specific to base classes.
@@ -101,6 +108,8 @@ def test_dump(self, pdepfile):
     def test_corrupted_file(self, pdepfile):
         if pdepfile.whichdb is None: # pragma: no cover
             pytest.skip('dumbdbm too dumb to detect db corruption')
+        if pdepfile.whichdb is 'redis': # pragma: no cover
+            pytest.skip('redis too dumb to detect db corruption')
 
         # create some corrupted files
         for name_ext in pdepfile.name_ext: