Skip to content
Browse files

Use taskqueue for quicker cleanup

git-svn-id: https://potato.codebasehq.com/getfavicon/getfavicon.svn/trunk@30 0c1c7bd3-9391-4859-b94e-4e2dcdc6a516
  • Loading branch information...
1 parent a4486e1 commit d1bcbffe72e325e3001a5053884a8912d2cbce7f jason committed Nov 6, 2010
Showing with 24 additions and 6 deletions.
  1. +21 −6 main.py
  2. +3 −0 queue.yaml
View
27 main.py
@@ -6,6 +6,7 @@
from google.appengine.api import memcache
from google.appengine.api import urlfetch
from google.appengine.ext.db import stats
+from google.appengine.api.labs import taskqueue
from urlparse import urlparse
from urlparse import urljoin
@@ -51,22 +52,35 @@ def get(self):
memcache.flush_all()
allfavIconQuery = favIcon.all()
- favIcons = allfavIconQuery.fetch(500)
+ favIcons = allfavIconQuery.fetch(250)
db.delete(favIcons)
class cleanup(BaseHandler):
def get(self):
- # Cleanup DS cache
- iconCacheCleanQuery = favIcon.gql("where dateCreated < :1",datetime.now()-timedelta(days=DS_CACHE_TIME))
- iconCacheCleanResults = iconCacheCleanQuery.fetch(500)
- db.delete(iconCacheCleanResults)
+ for i in range(25):
+ taskqueue.add(
+ queue_name='doCleanup',
+ url='/_doCleanup',
+ method='GET'
+ )
# Update Counts
counter.UpdateDSCounters()
-
+
+
+class doCleanup(BaseHandler):
+
+ def get(self):
+
+ # Cleanup DS cache
+ iconCacheCleanQuery = favIcon.gql("where dateCreated < :1",datetime.now()-timedelta(days=DS_CACHE_TIME))
+ iconCacheCleanResults = iconCacheCleanQuery.fetch(200)
+ db.delete(iconCacheCleanResults)
+ inf("Deleted %d old icon caches in DS" % len(iconCacheCleanResults))
+
class IndexPage(BaseHandler):
@@ -452,6 +466,7 @@ def main():
('/decache/', Decache),
('/test/', TestPage),
('/_cleanup', cleanup),
+ ('/_doCleanup', doCleanup),
('/_deleteall', deleteAll),
('/.*', PrintFavicon),
],
View
3 queue.yaml
@@ -0,0 +1,3 @@
+queue:
+- name: doCleanup
+ rate: 1/s

0 comments on commit d1bcbff

Please sign in to comment.
Something went wrong with that request. Please try again.