Merge branch 'bug817'

* bug817:
  implement the changeHorizon

master/buildbot/changes/manager.py

@@ -35,6 +35,8 @@
 #
 # ***** END LICENSE BLOCK *****
 
+import time
+
 from zope.interface import implements
 from twisted.python import log
 from twisted.internet import defer
@@ -78,12 +80,15 @@ class ChangeManager(service.MultiService):
 
     implements(interfaces.IEventSource)
 
-    changeHorizon = 0
+    changeHorizon = None
+    lastPruneChanges = None
     name = "changemanager"
 
     def __init__(self):
         service.MultiService.__init__(self)
         self._cache = util.LRUCache()
+        self.lastPruneChanges = 0
+        self.changeHorizon = 0
 
     def addSource(self, source):
         assert interfaces.IChangeSource.providedBy(source)
@@ -103,16 +108,26 @@ def addChange(self, change):
                                               change.comments, change.category))
         log.msg(msg.encode('utf-8', 'replace'))
 
-        #self.pruneChanges() # use self.changeHorizon
-        # for now, add these in the background, without waiting for it. TODO:
-        # return a Deferred.
-        #self.queue.add(db.runInteraction, self.addChangeToDatabase, change)
-
         # this sets change.number, if it wasn't already set (by the
         # migration-from-pickle code). It also fires a notification which
         # wakes up the Schedulers.
         self.parent.addChange(change)
 
+        self.pruneChanges(change.number)
+
+    def pruneChanges(self, last_added_changeid):
+        # this is an expensive operation, so only do it once per second, in case
+        # addChanges is called frequently
+        print "HERE, last =", last_added_changeid
+        if not self.changeHorizon or self.lastPruneChanges > time.time() - 1:
+            return
+        self.lastPruneChanges = time.time()
+
+        ids = self.parent.db.getChangeIdsLessThanIdNow(last_added_changeid - self.changeHorizon + 1)
+        print "ids", ids
+        for changeid in ids:
+            log.msg("removing change with id %s" % changeid)
+            self.parent.db.removeChangeNow(changeid)
 
     # IEventSource methods
 

master/buildbot/db/connector.py

@@ -494,6 +494,27 @@ def _txn_getChangesGreaterThan(self, t, last_changeid):
         changes.sort(key=lambda c: c.number)
         return changes
 
+    def getChangeIdsLessThanIdNow(self, new_changeid):
+        """Return a list of all extant change id's less than the given value,
+        sorted by number."""
+        def txn(t):
+            q = self.quoteq("SELECT changeid FROM changes WHERE changeid < ?")
+            t.execute(q, (new_changeid,))
+            changes = [changeid for (changeid,) in t.fetchall()]
+            changes.sort()
+            return changes
+        return self.runInteractionNow(txn)
+
+    def removeChangeNow(self, changeid):
+        """Thoroughly remove a change from the database, including all dependent
+        tables"""
+        def txn(t):
+            for table in ('changes', 'scheduler_changes', 'sourcestamp_changes',
+                          'change_files', 'change_links', 'change_properties'):
+                q = self.quoteq("DELETE FROM %s WHERE changeid = ?" % table)
+                t.execute(q, (changeid,))
+        return self.runInteractionNow(txn)
+
     def getChangesByNumber(self, changeids):
         return defer.gatherResults([self.getChangeByNumber(changeid)
                                     for changeid in changeids])