add removing builds and builders

master/buildbot/db/build_data.py

@@ -142,29 +142,30 @@ def thd(conn):
 
             if horizon_per_builder is not None:
                 for builderName in horizon_per_builder:
-                    older_than_timestamp_ = datetime2epoch(datetime.now() -
-                        horizon_per_builder[builderName]["buildDataHorizon"])
-                    if self.db._engine.dialect.name == 'sqlite':
-                        # sqlite does not support delete with a join,
-                        # so for this case we use a subquery,
-                        # which is much slower
-
-                        q = sa.select([builds.c.id])
-                        q = q.where(sa.and_((builds.c.complete_at < older_than_timestamp_),
-                                    builds.c.builderid == builders.c.id,
-                                    builders.c.name.like(builderName)))
-
-                        q = build_data.delete().where(build_data.c.buildid.in_(q))
-
-                    else:
-                        q = build_data.delete()
-                        q = q.where(sa.and_(builds.c.id == build_data.c.buildid,
-                                    builds.c.builderid == builders.c.id,
-                                    builds.c.complete_at < older_than_timestamp_,
-                                    builders.c.name.like(builderName)))
-
-                    res = conn.execute(q)
-                    res.close()
+                    if "buildDataHorizon" in horizon_per_builder[builderName]:
+                        older_than_timestamp_ = datetime2epoch(datetime.now() -
+                            horizon_per_builder[builderName]["buildDataHorizon"])
+                        if self.db._engine.dialect.name == 'sqlite':
+                            # sqlite does not support delete with a join,
+                            # so for this case we use a subquery,
+                            # which is much slower
+
+                            q = sa.select([builds.c.id])
+                            q = q.where(sa.and_((builds.c.complete_at < older_than_timestamp_),
+                                        builds.c.builderid == builders.c.id,
+                                        builders.c.name.like(builderName)))
+
+                            q = build_data.delete().where(build_data.c.buildid.in_(q))
+
+                        else:
+                            q = build_data.delete()
+                            q = q.where(sa.and_(builds.c.id == build_data.c.buildid,
+                                        builds.c.builderid == builders.c.id,
+                                        builds.c.complete_at < older_than_timestamp_,
+                                        builders.c.name.like(builderName)))
+
+                        res = conn.execute(q)
+                        res.close()
             else:
                 if self.db._engine.dialect.name == 'sqlite':
                     # sqlite does not support delete with a join,

master/buildbot/db/builders.py

@@ -21,7 +21,8 @@
 from twisted.internet import defer
 
 from buildbot.db import base
-
+from buildbot.util import datetime2epoch
+from datetime import datetime
 
 class BuildersConnectorComponent(base.DBConnectorComponent):
 
@@ -149,3 +150,32 @@ def thd(conn):
                     last['masterids'].append(row['masterid'])
             return rv
         return self.db.pool.do(thd)
+
+    # returns a Deferred that returns a value
+    def deleteOldBuilders(self):
+        builders = self.db.model.builders
+        builds = self.db.model.builds
+        builder_masters = self.db.model.builder_masters
+
+        def countBuilders(conn):
+            res = conn.execute(sa.select([sa.func.count(builders.c.id)]))
+            count = res.fetchone()[0]
+            res.close()
+            return count
+
+        def thdDeleteOldBuilders(conn):
+            count_before = countBuilders(conn)
+
+            q1 = sa.select([builder_masters.c.builderid])
+            q2 = sa.select([builds.c.builderid])
+            q = q1.union(q2)
+            q = builders.delete().where(builders.c.id.notin_(q))
+
+            res = conn.execute(q)
+            res.close()
+
+            count_after = countBuilders(conn)
+
+            return count_before - count_after
+
+        return self.db.pool.do(thdDeleteOldBuilders)

master/buildbot/db/builds.py

@@ -23,7 +23,8 @@
 from buildbot.db import NULL
 from buildbot.db import base
 from buildbot.util import epoch2datetime
-
+from buildbot.util import datetime2epoch
+from datetime import datetime
 
 class BuildsConnectorComponent(base.DBConnectorComponent):
     # Documentation is in developer/db.rst
@@ -257,3 +258,47 @@ def _builddictFromRow(self, row):
             complete_at=epoch2datetime(row.complete_at),
             state_string=row.state_string,
             results=row.results)
+
+    # returns a Deferred that returns a value
+    def deleteOldBuilds(self, horizon_per_builder=None):
+        builds = self.db.model.builds
+        builders = self.db.model.builders
+
+        def countBuilds(conn):
+            res = conn.execute(sa.select([sa.func.count(builds.c.id)]))
+            count = res.fetchone()[0]
+            res.close()
+            return count
+
+        def thdDeleteOldBuilds(conn):
+            count_before = countBuilds(conn)
+
+            for builderName in horizon_per_builder:
+                if "buildHorizon" in horizon_per_builder[builderName]:
+                    older_than_timestamp = datetime2epoch(datetime.now() -
+                        horizon_per_builder[builderName]["buildHorizon"])
+                    if self.db._engine.dialect.name == 'sqlite':
+                        # sqlite does not support delete with a join,
+                        # so for this case we use a subquery,
+                        # which is much slower
+
+                        q = sa.select([builds.c.id])
+                        q = q.where(sa.and_((builds.c.builderid == builders.c.id,
+                                             builds.c.complete_at < older_than_timestamp),                                
+                                             builders.c.name.like(builderName)))
+                        q = builds.delete().where(builds.c.id.in_(q))
+
+                    else:
+                        q = builds.delete()
+                        q = q.where(sa.and_(builds.c.builderid == builders.c.id,
+                                            builds.c.complete_at < older_than_timestamp,
+                                            builders.c.name.like(builderName)))
+
+                    res = conn.execute(q)
+                    res.close()
+
+            count_after = countBuilds(conn)
+
+            return count_before - count_after
+
+        return self.db.pool.do(thdDeleteOldBuilds)

master/buildbot/db/logs.py

@@ -404,22 +404,23 @@ def thddeleteOldLogs(conn):
 
             if horizon_per_builder:
                 for builderName in horizon_per_builder:
-                    older_than_timestamp_ = datetime2epoch(datetime.now() -
-                        horizon_per_builder[builderName]["logHorizon"])
-                    stepid_max = getStepidMax(conn, older_than_timestamp_)
-                    if stepid_max:
-                        subquery = sa.select([model.steps.c.id]).where(
-                            sa.and_(model.steps.c.buildid == model.builds.c.id,
-                            model.builds.c.builderid == model.builders.c.id,
-                            model.builders.c.name.like(builderName)))
-                        res = conn.execute(
-                            model.logs.update()
-                            .where(sa.and_(model.logs.c.stepid.in_(subquery),
-                                           model.logs.c.stepid <= stepid_max,
-                                           model.logs.c.type != 'd'))
-                            .values(type='d')
-                        )
-                        res.close()
+                    if "logHorizon" in horizon_per_builder[builderName]:
+                        older_than_timestamp_ = datetime2epoch(datetime.now() -
+                            horizon_per_builder[builderName]["logHorizon"])
+                        stepid_max = getStepidMax(conn, older_than_timestamp_)
+                        if stepid_max:
+                            subquery = sa.select([model.steps.c.id]).where(
+                                sa.and_(model.steps.c.buildid == model.builds.c.id,
+                                model.builds.c.builderid == model.builders.c.id,
+                                model.builders.c.name.like(builderName)))
+                            res = conn.execute(
+                                model.logs.update()
+                                .where(sa.and_(model.logs.c.stepid.in_(subquery),
+                                               model.logs.c.stepid <= stepid_max,
+                                               model.logs.c.type != 'd'))
+                                .values(type='d')
+                            )
+                            res.close()
             else:
                 stepid_max = getStepidMax(conn, older_than_timestamp)
                 if stepid_max:
@@ -436,8 +437,9 @@ def thddeleteOldLogs(conn):
             deleteLogsWithTypeD(conn)
 
             count_after = countLogchunks(conn)
+            count = count_before - count_after 
 
-            return count_before - count_after
+            return count if count > 0 else 0
         return self.db.pool.do(thddeleteOldLogs)
 
     def _logdictFromRow(self, row):