Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Fixed #6464 -- Added incr() and decr() operations on cache backends. …

…Atomic on Memcache; implemented as a 2 stage retrieve/update on other backends. Includes refactor of the cache tests to ensure all the backends are actually tested, and a fix to the DB cache backend that was discovered as a result. Thanks to Michael Malone for the original patch.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@10031 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit 638dbc3e8320652ba3bd759e0ac1d28590402096 1 parent 1d8e6ea
Russell Keith-Magee authored March 11, 2009
18  django/core/cache/backends/base.py
@@ -65,6 +65,24 @@ def has_key(self, key):
65 65
         """
66 66
         return self.get(key) is not None
67 67
 
  68
+    def incr(self, key, delta=1):
  69
+        """
  70
+        Add delta to value in the cache. If the key does not exist, raise a
  71
+        ValueError exception.
  72
+        """
  73
+        if key not in self:
  74
+            raise ValueError, "Key '%s' not found" % key
  75
+        new_value = self.get(key) + delta
  76
+        self.set(key, new_value)
  77
+        return new_value
  78
+
  79
+    def decr(self, key, delta=1):
  80
+        """
  81
+        Subtract delta from value in the cache. If the key does not exist, raise
  82
+        a ValueError exception.
  83
+        """
  84
+        return self.incr(key, -delta)
  85
+
68 86
     def __contains__(self, key):
69 87
         """
70 88
         Returns True if the key is in the cache and has not expired.
1  django/core/cache/backends/db.py
@@ -64,6 +64,7 @@ def _base_set(self, mode, key, value, timeout=None):
64 64
                 cursor.execute("INSERT INTO %s (cache_key, value, expires) VALUES (%%s, %%s, %%s)" % self._table, [key, encoded, str(exp)])
65 65
         except DatabaseError:
66 66
             # To be threadsafe, updates/inserts are allowed to fail silently
  67
+            transaction.rollback()
67 68
             return False
68 69
         else:
69 70
             transaction.commit_unless_managed()
5  django/core/cache/backends/memcached.py
@@ -45,3 +45,8 @@ def get_many(self, keys):
45 45
     def close(self, **kwargs):
46 46
         self._cache.disconnect_all()
47 47
 
  48
+    def incr(self, key, delta=1):
  49
+        return self._cache.incr(key, delta)
  50
+
  51
+    def decr(self, key, delta=1):
  52
+        return self._cache.decr(key, delta)
43  docs/internals/contributing.txt
@@ -184,7 +184,7 @@ Patch style
184 184
       An exception is for code changes that are described more clearly in plain
185 185
       English than in code. Indentation is the most common example; it's hard to
186 186
       read patches when the only difference in code is that it's indented.
187  
-      
  187
+
188 188
       Patches in ``git diff`` format are also acceptable.
189 189
 
190 190
     * When creating patches, always run ``svn diff`` from the top-level
@@ -402,7 +402,7 @@ translated, here's what to do:
402 402
 
403 403
     * Join the `Django i18n mailing list`_ and introduce yourself.
404 404
 
405  
-    * Create translations using the methods described in the 
  405
+    * Create translations using the methods described in the
406 406
       :ref:`i18n documentation <topics-i18n>`. For this you will use the
407 407
       ``django-admin.py makemessages`` tool. In this particular case it should
408 408
       be run from the top-level ``django`` directory of the Django source tree.
@@ -697,9 +697,9 @@ repository:
697 697
       first commit the change to library Y, then commit feature X in a separate
698 698
       commit. This goes a *long way* in helping all core Django developers
699 699
       follow your changes.
700  
-      
701  
-    * Separate bug fixes from feature changes. 
702  
-    
  700
+
  701
+    * Separate bug fixes from feature changes.
  702
+
703 703
       Bug fixes need to be added to the current bugfix branch (e.g. the
704 704
       ``1.0.X`` branch) as well as the current trunk.
705 705
 
@@ -782,6 +782,10 @@ dependencies:
782 782
     *  Textile_
783 783
     *  Docutils_
784 784
     *  setuptools_
  785
+    *  memcached_, plus the either the python-memcached_ or cmemcached_ Python binding
  786
+
  787
+If you want to test the memcached cache backend, you will also need to define
  788
+a :setting:`CACHE_BACKEND` setting that points at your memcached instance.
785 789
 
786 790
 Each of these dependencies is optional. If you're missing any of them, the
787 791
 associated tests will be skipped.
@@ -791,6 +795,9 @@ associated tests will be skipped.
791 795
 .. _Textile: http://pypi.python.org/pypi/textile
792 796
 .. _docutils: http://pypi.python.org/pypi/docutils/0.4
793 797
 .. _setuptools: http://pypi.python.org/pypi/setuptools/
  798
+.. _memcached: http://www.danga.com/memcached/
  799
+.. _python-memcached: http://pypi.python.org/pypi/python-memcached/
  800
+.. _cmemcached: http://pypi.python.org/pypi/cmemcache
794 801
 
795 802
 To run a subset of the unit tests, append the names of the test modules to the
796 803
 ``runtests.py`` command line. See the list of directories in
@@ -862,28 +869,28 @@ for feature branches:
862 869
 
863 870
     1. Feature branches using a distributed revision control system like
864 871
        Git_, Mercurial_, Bazaar_, etc.
865  
-       
866  
-       If you're familiar with one of these tools, this is probably your best 
  872
+
  873
+       If you're familiar with one of these tools, this is probably your best
867 874
        option since it doesn't require any support or buy-in from the Django
868 875
        core developers.
869  
-       
  876
+
870 877
        However, do keep in mind that Django will continue to use Subversion for
871 878
        the foreseeable future, and this will naturally limit the recognition of
872 879
        your branch. Further, if your branch becomes eligible for merging to
873 880
        trunk you'll need to find a core developer familiar with your DVCS of
874 881
        choice who'll actually perform the merge.
875  
-       
  882
+
876 883
        If you do decided to start a distributed branch of Django and choose to make it
877 884
        public, please add the branch to the `Django branches`_ wiki page.
878  
-       
  885
+
879 886
     2. Feature branches using SVN have a higher bar. If you want a branch in SVN
880 887
        itself, you'll need a "mentor" among the :ref:`core committers
881 888
        <internals-committers>`. This person is responsible for actually creating
882 889
        the branch, monitoring your process (see below), and ultimately merging
883 890
        the branch into trunk.
884  
-       
  891
+
885 892
        If you want a feature branch in SVN, you'll need to ask in
886  
-       `django-developers`_ for a mentor. 
  893
+       `django-developers`_ for a mentor.
887 894
 
888 895
 .. _git: http://git.or.cz/
889 896
 .. _mercurial: http://www.selenic.com/mercurial/
@@ -894,7 +901,7 @@ Branch rules
894 901
 ------------
895 902
 
896 903
 We've got a few rules for branches born out of experience with what makes a
897  
-successful Django branch. 
  904
+successful Django branch.
898 905
 
899 906
 DVCS branches are obviously not under central control, so we have no way of
900 907
 enforcing these rules. However, if you're using a DVCS, following these rules
@@ -908,19 +915,19 @@ rules are broken.
908 915
     * Only branch entire copies of the Django tree, even if work is only
909 916
       happening on part of that tree. This makes it painless to switch to a
910 917
       branch.
911  
-      
  918
+
912 919
     * Merge changes from trunk no less than once a week, and preferably every
913 920
       couple-three days.
914  
-      
  921
+
915 922
       In our experience, doing regular trunk merges is often the difference
916 923
       between a successful branch and one that fizzles and dies.
917  
-      
  924
+
918 925
       If you're working on an SVN branch, you should be using `svnmerge.py`_
919 926
       to track merges from trunk.
920  
-      
  927
+
921 928
     * Keep tests passing and documentation up-to-date. As with patches,
922 929
       we'll only merge a branch that comes with tests and documentation.
923  
-      
  930
+
924 931
 .. _svnmerge.py: http://www.orcaware.com/svn/wiki/Svnmerge.py
925 932
 
926 933
 Once the branch is stable and ready to be merged into the trunk, alert
29  docs/topics/cache.txt
@@ -162,7 +162,7 @@ cache is multi-process and thread-safe. To use it, set ``CACHE_BACKEND`` to
162 162
 ``"locmem:///"``. For example::
163 163
 
164 164
     CACHE_BACKEND = 'locmem:///'
165  
-    
  165
+
166 166
 Note that each process will have its own private cache instance, which means no
167 167
 cross-process caching is possible. This obviously also means the local memory
168 168
 cache isn't particularly memory-efficient, so it's probably not a good choice
@@ -439,6 +439,33 @@ of clearing the cache for a particular object::
439 439
 
440 440
     >>> cache.delete('a')
441 441
 
  442
+.. versionadded:: 1.1
  443
+
  444
+You can also increment or decrement a key that already exists using the
  445
+``incr()`` or ``decr()`` methods, respectively. By default, the existing cache
  446
+value will incremented or decremented by 1. Other increment/decrement values
  447
+can be specified by providing an argument to the increment/decrement call. A
  448
+ValueError will be raised if you attempt to increment or decrement a
  449
+nonexistent cache key.::
  450
+
  451
+    >>> cache.set('num', 1)
  452
+    >>> cache.incr('num')
  453
+    2
  454
+    >>> cache.incr('num', 10)
  455
+    12
  456
+    >>> cache.decr('num')
  457
+    11
  458
+    >>> cache.decr('num', 5)
  459
+    6
  460
+
  461
+.. note::
  462
+
  463
+    ``incr()``/``decr()`` methods are not guaranteed to be atomic. On those
  464
+    backends that support atomic increment/decrement (most notably, the
  465
+    memcached backend), increment and decrement operations will be atomic.
  466
+    However, if the backend doesn't natively provide an increment/decrement
  467
+    operation, it will be implemented using a 2 step retrieve/update.
  468
+
442 469
 That's it. The cache has very few restrictions: You can cache any object that
443 470
 can be pickled safely, although keys must be strings.
444 471
 
184  tests/regressiontests/cache/tests.py
@@ -9,8 +9,10 @@
9 9
 import time
10 10
 import unittest
11 11
 
12  
-from django.core.cache import cache, get_cache
13  
-from django.core.cache.backends.filebased import CacheClass as FileCache
  12
+from django.conf import settings
  13
+from django.core import management
  14
+from django.core.cache import get_cache
  15
+from django.core.cache.backends.base import InvalidCacheBackendError
14 16
 from django.http import HttpResponse
15 17
 from django.utils.cache import patch_vary_headers
16 18
 from django.utils.hashcompat import md5_constructor
@@ -22,39 +24,133 @@ class C:
22 24
     def m(n):
23 25
         return 24
24 26
 
25  
-class Cache(unittest.TestCase):
  27
+class DummyCacheTests(unittest.TestCase):
  28
+    # The Dummy cache backend doesn't really behave like a test backend,
  29
+    # so it has different test requirements.
26 30
     def setUp(self):
27  
-        # Special-case the file cache so we can clean up after ourselves.
28  
-        if isinstance(cache, FileCache):
29  
-            self.cache_dir = tempfile.mkdtemp()
30  
-            self.cache = get_cache("file:///%s" % self.cache_dir)
31  
-        else:
32  
-            self.cache_dir = None
33  
-            self.cache = cache
34  
-            
35  
-    def tearDown(self):
36  
-        if self.cache_dir is not None:
37  
-            shutil.rmtree(self.cache_dir)
38  
-    
  31
+        self.cache = get_cache('dummy://')
  32
+
  33
+    def test_simple(self):
  34
+        "Dummy cache backend ignores cache set calls"
  35
+        self.cache.set("key", "value")
  36
+        self.assertEqual(self.cache.get("key"), None)
  37
+
  38
+    def test_add(self):
  39
+        "Add doesn't do anything in dummy cache backend"
  40
+        self.cache.add("addkey1", "value")
  41
+        result = self.cache.add("addkey1", "newvalue")
  42
+        self.assertEqual(result, True)
  43
+        self.assertEqual(self.cache.get("addkey1"), None)
  44
+
  45
+    def test_non_existent(self):
  46
+        "Non-existent keys aren't found in the dummy cache backend"
  47
+        self.assertEqual(self.cache.get("does_not_exist"), None)
  48
+        self.assertEqual(self.cache.get("does_not_exist", "bang!"), "bang!")
  49
+
  50
+    def test_get_many(self):
  51
+        "get_many returns nothing for the dummy cache backend"
  52
+        self.cache.set('a', 'a')
  53
+        self.cache.set('b', 'b')
  54
+        self.cache.set('c', 'c')
  55
+        self.cache.set('d', 'd')
  56
+        self.assertEqual(self.cache.get_many(['a', 'c', 'd']), {})
  57
+        self.assertEqual(self.cache.get_many(['a', 'b', 'e']), {})
  58
+
  59
+    def test_delete(self):
  60
+        "Cache deletion is transparently ignored on the dummy cache backend"
  61
+        self.cache.set("key1", "spam")
  62
+        self.cache.set("key2", "eggs")
  63
+        self.assertEqual(self.cache.get("key1"), None)
  64
+        self.cache.delete("key1")
  65
+        self.assertEqual(self.cache.get("key1"), None)
  66
+        self.assertEqual(self.cache.get("key2"), None)
  67
+
  68
+    def test_has_key(self):
  69
+        "The has_key method doesn't ever return True for the dummy cache backend"
  70
+        self.cache.set("hello1", "goodbye1")
  71
+        self.assertEqual(self.cache.has_key("hello1"), False)
  72
+        self.assertEqual(self.cache.has_key("goodbye1"), False)
  73
+
  74
+    def test_in(self):
  75
+        "The in operator doesn't ever return True for the dummy cache backend"
  76
+        self.cache.set("hello2", "goodbye2")
  77
+        self.assertEqual("hello2" in self.cache, False)
  78
+        self.assertEqual("goodbye2" in self.cache, False)
  79
+
  80
+    def test_incr(self):
  81
+        "Dummy cache values can't be incremented"
  82
+        self.cache.set('answer', 42)
  83
+        self.assertRaises(ValueError, self.cache.incr, 'answer')
  84
+        self.assertRaises(ValueError, self.cache.incr, 'does_not_exist')
  85
+
  86
+    def test_decr(self):
  87
+        "Dummy cache values can't be decremented"
  88
+        self.cache.set('answer', 42)
  89
+        self.assertRaises(ValueError, self.cache.decr, 'answer')
  90
+        self.assertRaises(ValueError, self.cache.decr, 'does_not_exist')
  91
+
  92
+    def test_data_types(self):
  93
+        "All data types are ignored equally by the dummy cache"
  94
+        stuff = {
  95
+            'string'    : 'this is a string',
  96
+            'int'       : 42,
  97
+            'list'      : [1, 2, 3, 4],
  98
+            'tuple'     : (1, 2, 3, 4),
  99
+            'dict'      : {'A': 1, 'B' : 2},
  100
+            'function'  : f,
  101
+            'class'     : C,
  102
+        }
  103
+        self.cache.set("stuff", stuff)
  104
+        self.assertEqual(self.cache.get("stuff"), None)
  105
+
  106
+    def test_expiration(self):
  107
+        "Expiration has no effect on the dummy cache"
  108
+        self.cache.set('expire1', 'very quickly', 1)
  109
+        self.cache.set('expire2', 'very quickly', 1)
  110
+        self.cache.set('expire3', 'very quickly', 1)
  111
+
  112
+        time.sleep(2)
  113
+        self.assertEqual(self.cache.get("expire1"), None)
  114
+
  115
+        self.cache.add("expire2", "newvalue")
  116
+        self.assertEqual(self.cache.get("expire2"), None)
  117
+        self.assertEqual(self.cache.has_key("expire3"), False)
  118
+
  119
+    def test_unicode(self):
  120
+        "Unicode values are ignored by the dummy cache"
  121
+        stuff = {
  122
+            u'ascii': u'ascii_value',
  123
+            u'unicode_ascii': u'Iñtërnâtiônàlizætiøn1',
  124
+            u'Iñtërnâtiônàlizætiøn': u'Iñtërnâtiônàlizætiøn2',
  125
+            u'ascii': {u'x' : 1 }
  126
+            }
  127
+        for (key, value) in stuff.items():
  128
+            self.cache.set(key, value)
  129
+            self.assertEqual(self.cache.get(key), None)
  130
+
  131
+
  132
+class BaseCacheTests(object):
  133
+    # A common set of tests to apply to all cache backends
39 134
     def test_simple(self):
40  
-        # simple set/get
  135
+        # Simple cache set/get works
41 136
         self.cache.set("key", "value")
42 137
         self.assertEqual(self.cache.get("key"), "value")
43 138
 
44 139
     def test_add(self):
45  
-        # test add (only add if key isn't already in cache)
  140
+        # A key can be added to a cache
46 141
         self.cache.add("addkey1", "value")
47 142
         result = self.cache.add("addkey1", "newvalue")
48 143
         self.assertEqual(result, False)
49 144
         self.assertEqual(self.cache.get("addkey1"), "value")
50 145
 
51 146
     def test_non_existent(self):
  147
+        # Non-existent cache keys return as None/default
52 148
         # get with non-existent keys
53 149
         self.assertEqual(self.cache.get("does_not_exist"), None)
54 150
         self.assertEqual(self.cache.get("does_not_exist", "bang!"), "bang!")
55 151
 
56 152
     def test_get_many(self):
57  
-        # get_many
  153
+        # Multiple cache keys can be returned using get_many
58 154
         self.cache.set('a', 'a')
59 155
         self.cache.set('b', 'b')
60 156
         self.cache.set('c', 'c')
@@ -63,7 +159,7 @@ def test_get_many(self):
63 159
         self.assertEqual(self.cache.get_many(['a', 'b', 'e']), {'a' : 'a', 'b' : 'b'})
64 160
 
65 161
     def test_delete(self):
66  
-        # delete
  162
+        # Cache keys can be deleted
67 163
         self.cache.set("key1", "spam")
68 164
         self.cache.set("key2", "eggs")
69 165
         self.assertEqual(self.cache.get("key1"), "spam")
@@ -72,17 +168,37 @@ def test_delete(self):
72 168
         self.assertEqual(self.cache.get("key2"), "eggs")
73 169
 
74 170
     def test_has_key(self):
75  
-        # has_key
  171
+        # The cache can be inspected for cache keys
76 172
         self.cache.set("hello1", "goodbye1")
77 173
         self.assertEqual(self.cache.has_key("hello1"), True)
78 174
         self.assertEqual(self.cache.has_key("goodbye1"), False)
79 175
 
80 176
     def test_in(self):
  177
+        # The in operator can be used to inspet cache contents
81 178
         self.cache.set("hello2", "goodbye2")
82 179
         self.assertEqual("hello2" in self.cache, True)
83 180
         self.assertEqual("goodbye2" in self.cache, False)
84 181
 
  182
+    def test_incr(self):
  183
+        # Cache values can be incremented
  184
+        self.cache.set('answer', 41)
  185
+        self.assertEqual(self.cache.incr('answer'), 42)
  186
+        self.assertEqual(self.cache.get('answer'), 42)
  187
+        self.assertEqual(self.cache.incr('answer', 10), 52)
  188
+        self.assertEqual(self.cache.get('answer'), 52)
  189
+        self.assertRaises(ValueError, self.cache.incr, 'does_not_exist')
  190
+
  191
+    def test_decr(self):
  192
+        # Cache values can be decremented
  193
+        self.cache.set('answer', 43)
  194
+        self.assertEqual(self.cache.decr('answer'), 42)
  195
+        self.assertEqual(self.cache.get('answer'), 42)
  196
+        self.assertEqual(self.cache.decr('answer', 10), 32)
  197
+        self.assertEqual(self.cache.get('answer'), 32)
  198
+        self.assertRaises(ValueError, self.cache.decr, 'does_not_exist')
  199
+
85 200
     def test_data_types(self):
  201
+        # Many different data types can be cached
86 202
         stuff = {
87 203
             'string'    : 'this is a string',
88 204
             'int'       : 42,
@@ -96,6 +212,7 @@ def test_data_types(self):
96 212
         self.assertEqual(self.cache.get("stuff"), stuff)
97 213
 
98 214
     def test_expiration(self):
  215
+        # Cache values can be set to expire
99 216
         self.cache.set('expire1', 'very quickly', 1)
100 217
         self.cache.set('expire2', 'very quickly', 1)
101 218
         self.cache.set('expire3', 'very quickly', 1)
@@ -108,6 +225,7 @@ def test_expiration(self):
108 225
         self.assertEqual(self.cache.has_key("expire3"), False)
109 226
 
110 227
     def test_unicode(self):
  228
+        # Unicode values can be cached
111 229
         stuff = {
112 230
             u'ascii': u'ascii_value',
113 231
             u'unicode_ascii': u'Iñtërnâtiônàlizætiøn1',
@@ -118,14 +236,36 @@ def test_unicode(self):
118 236
             self.cache.set(key, value)
119 237
             self.assertEqual(self.cache.get(key), value)
120 238
 
  239
+class DBCacheTests(unittest.TestCase, BaseCacheTests):
  240
+    def setUp(self):
  241
+        management.call_command('createcachetable', 'test_cache_table', verbosity=0, interactive=False)
  242
+        self.cache = get_cache('db://test_cache_table')
  243
+
  244
+    def tearDown(self):
  245
+        from django.db import connection
  246
+        cursor = connection.cursor()
  247
+        cursor.execute('DROP TABLE test_cache_table');
  248
+
  249
+class LocMemCacheTests(unittest.TestCase, BaseCacheTests):
  250
+    def setUp(self):
  251
+        self.cache = get_cache('locmem://')
  252
+
  253
+# memcached backend isn't guaranteed to be available.
  254
+# To check the memcached backend, the test settings file will
  255
+# need to contain a CACHE_BACKEND setting that points at
  256
+# your memcache server.
  257
+if settings.CACHE_BACKEND.startswith('memcached://'):
  258
+    class MemcachedCacheTests(unittest.TestCase, BaseCacheTests):
  259
+        def setUp(self):
  260
+            self.cache = get_cache(settings.CACHE_BACKEND)
121 261
 
122  
-class FileBasedCacheTests(unittest.TestCase):
  262
+class FileBasedCacheTests(unittest.TestCase, BaseCacheTests):
123 263
     """
124 264
     Specific test cases for the file-based cache.
125 265
     """
126 266
     def setUp(self):
127 267
         self.dirname = tempfile.mkdtemp()
128  
-        self.cache = FileCache(self.dirname, {})
  268
+        self.cache = get_cache('file:///%s' % self.dirname)
129 269
 
130 270
     def tearDown(self):
131 271
         shutil.rmtree(self.dirname)

0 notes on commit 638dbc3

Please sign in to comment.
Something went wrong with that request. Please try again.