DocSavage / sharded_counter

Sharded, memcached counter for Google App Engine

This URL has Read+Write access

sharded_counter / counter.py
raw normal view history
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 1 # Copyright 2008 William T Katz
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 #
15 # THIS LICENSE INFORMATION/ATTRIBUTION must be left in place.
16
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 17 import string
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 18 import random
19 import logging
20
21 from google.appengine.api import memcache
22 from google.appengine.ext import db
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 23 from google.appengine.runtime import apiproxy_errors
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 24
25
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 26 class MemcachedCount(object):
27 DELTA_ZERO = 500000 # Allows negative numbers in unsigned memcache
28
29 def __init__(self, name):
30 self.key = 'MemcachedCount' + name
31
32 def get_count(self):
33 value = memcache.get(self.key)
34 if value is None:
35 return 0
36 else:
37 return string.atoi(value) - MemcachedCount.DELTA_ZERO
38
39 def set_count(self, value):
40 memcache.set(self.key, str(MemcachedCount.DELTA_ZERO + value))
41
42 def delete_count(self):
43 memcache.delete(self.key)
44
45 count = property(get_count, set_count, delete_count)
46
47 def increment(self, incr=1):
48 value = memcache.get(self.key)
49 if value is None:
50 self.count = incr
51 elif incr > 0:
52 memcache.incr(self.key, incr)
53 elif incr < 0:
54 memcache.decr(self.key, -incr)
55
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 56 class Counter(object):
57 """A counter using sharded writes to prevent contentions.
58
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 59 Should be used for counters that handle a lot of concurrent use.
60 Follows pattern described in Google I/O talk:
61 http://sites.google.com/site/io/building-scalable-web-applications-with-google-app-engine
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 62
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 63 Memcache is used for caching counts and if a cached count is available, it is
64 the most correct. If there are datastore put issues, we store the un-put values
65 into a delayed_incr memcache that will be applied as soon as the next shard put
66 is successful. Changes will only be lost if we lose memcache before a successful
67 datastore shard put or there's a failure/error in memcache.
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 68
69 Usage:
70 hits = Counter('hits')
71 hits.increment()
72 my_hits = hits.count
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 73 hits.get_count(nocache=True) # Forces non-cached count of all shards
74 hits.count = 6 # Set the counter to arbitrary value
75 hits.increment(incr=-1) # Decrement
76 hits.increment(10)
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 77 """
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 78 NUM_SHARDS = 20
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 79
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 80 def __init__(self, name):
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 81 self.name = name
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 82 self.memcached = MemcachedCount('Counter' + name)
83 self.delayed_incr = MemcachedCount('DelayedIncr' + name)
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 84
85 def delete(self):
86 q = db.Query(CounterShard).filter('name =', self.name)
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 87 shards = q.fetch(limit=Counter.NUM_SHARDS)
88 db.delete(shards)
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 89
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 90 def get_count_and_cache(self):
91 q = db.Query(CounterShard).filter('name =', self.name)
92 shards = q.fetch(limit=Counter.NUM_SHARDS)
93 datastore_count = 0
94 for shard in shards:
95 datastore_count += shard.count
96 count = datastore_count + self.delayed_incr.count
97 self.memcached.count = count
98 return count
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 99
100 def get_count(self, nocache=False):
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 101 total = self.memcached.count
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 102 if nocache or total is None:
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 103 return self.get_count_and_cache()
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 104 else:
105 return int(total)
106
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 107 def set_count(self, value):
108 cur_value = self.get_count()
109 self.memcached.count = value
110 delta = value - cur_value
111 if delta != 0:
112 CounterShard.increment(self, incr=delta)
1175a75b » DocSavage 2008-10-07 Remove unnecessary shard na... 113
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 114 count = property(get_count, set_count)
115
116 def increment(self, incr=1, refresh=False):
117 CounterShard.increment(self, incr)
118 self.memcached.increment(incr)
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 119
120
121 class CounterShard(db.Model):
122 name = db.StringProperty(required=True)
123 count = db.IntegerProperty(default=0)
124
125 @classmethod
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 126 def increment(cls, counter, incr=1):
127 index = random.randint(1, Counter.NUM_SHARDS)
128 counter_name = counter.name
129 delayed_incr = counter.delayed_incr.count
1175a75b » DocSavage 2008-10-07 Remove unnecessary shard na... 130 shard_key_name = 'Shard' + counter_name + str(index)
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 131 def get_or_create_shard():
132 shard = CounterShard.get_by_key_name(shard_key_name)
133 if shard is None:
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 134 shard = CounterShard(key_name=shard_key_name, name=counter_name)
135 shard.count += incr + delayed_incr
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 136 key = shard.put()
137 try:
138 db.run_in_transaction(get_or_create_shard)
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 139 except (db.Error, apiproxy_errors.Error), e:
140 counter.delayed_incr.increment(incr)
141 logging.error("CounterShard (%s) delayed increment %d: %s",
142 counter_name, incr, e)
14d2e5b3 » DocSavage 2008-09-11 Spun off from Bloog 143 return False
c5ec5d99 » DocSavage 2008-10-07 Cleaned up code. Added abi... 144 if delayed_incr:
145 counter.delayed_incr.count = 0
146 return True