Merge branch 'stats-service' of https://github.com/prasoon2211/buildbot…

… into pr1725

master/buildbot/plugins/__init__.py

@@ -22,9 +22,10 @@
 from buildbot.interfaces import IChangeSource
 from buildbot.interfaces import IScheduler
 from buildbot.plugins.db import get_plugins
+from buildbot import statistics as stats
 
 
-__all__ = ['changes', 'schedulers', 'buildslave', 'steps', 'util', 'reporters']
+__all__ = ['changes', 'schedulers', 'buildslave', 'steps', 'util', 'reporters', 'stats']
 
 
 # Names here match the names of the corresponding Buildbot module, hence

master/buildbot/process/build.py

@@ -301,7 +301,6 @@ def startBuild(self, build_status, expectations, slavebuilder):
         yield self.master.data.updates.setBuildStateString(self.buildid,
                                                            u'finished')
         yield self.master.data.updates.finishBuild(self.buildid, self.results)
-
         # mark the build as finished
         self.slavebuilder.buildFinished()
         slave.updateSlaveStatus(buildFinished=self)

master/buildbot/statistics/__init__.py

@@ -0,0 +1,27 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from buildbot.statistics.stats_service import StatsService
+from buildbot.statistics.storage_backends import InfluxStorageService
+from buildbot.statistics.capture import CaptureProperty
+from buildbot.statistics.capture import CaptureBuildDuration
+from buildbot.statistics.capture import CaptureBuildStartTime
+from buildbot.statistics.capture import CaptureBuildEndTime
+from buildbot.statistics.capture import CaptureData
+
+__all__ = [
+    'StatsService', 'InfluxStorageService', 'CaptureProperty', 'CaptureBuildDuration',
+    'CaptureBuildStartTime', 'CaptureBuildEndTime', 'CaptureData'
+]

master/buildbot/statistics/capture.py

@@ -0,0 +1,200 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from twisted.internet import defer
+from twisted.internet import threads
+
+
+class Capture(object):
+    """
+    Base class for all Capture* classes.
+    """
+    def __init__(self, routingKey, callback):
+        self.routingKey = routingKey
+        self.callback = callback
+        # parent service and buildmaster to be set when StatsService initialized
+        self.parent_svcs = []
+        self.master = None
+
+    def defaultContext(self, msg):
+        return {
+            "builder_name": self.builder_name,
+            "build_number": str(msg['number'])
+        }
+
+    def consumer(self, routingKey, msg):
+        raise NotImplementedError
+
+
+class CaptureProperty(Capture):
+    """
+    Convenience wrapper for getting statistics for filtering.
+    Filters out build properties specifies in the config file.
+    """
+    def __init__(self, builder_name, property_name, callback=None):
+        self.builder_name = builder_name
+        self.property_name = property_name
+        routingKey = ("builders", None, "builds", None, "finished")
+
+        def default_callback(props, property_name):
+            return props[property_name][0]  # index: 0 - prop_value, 1 - prop_source
+
+        if not callback:
+            callback = default_callback
+
+        Capture.__init__(self, routingKey, callback)
+
+    @defer.inlineCallbacks
+    def consumer(self, routingKey, msg):
+        """
+        Consumer for this (CaptureProperty) class. Gets the properties from data api and
+        send them to the storage backends.
+        """
+        builder_info = yield self.master.data.get(("builders", msg['builderid']))
+        if self.builder_name == builder_info['name']:
+            properties = yield self.master.data.get(("builds", msg['buildid'], "properties"))
+            ret_val = self.callback(properties, self.property_name)
+            context = self.defaultContext(msg)
+            series_name = self.builder_name + "-" + self.property_name
+            post_data = {
+                "name": self.property_name,
+                "value": ret_val
+            }
+            for svc in self.parent_svcs:
+                yield threads.deferToThread(svc.postStatsValue, post_data, series_name,
+                                            context)
+
+        else:
+            yield defer.succeed(None)
+
+
+class CaptureBuildTimes(Capture):
+    """
+    Capture methods for capturing build start times.
+    """
+    def __init__(self, builder_name, callback):
+        self.builder_name = builder_name
+        routingKey = ("builders", None, "builds", None, "finished")
+        Capture.__init__(self, routingKey, callback)
+
+    @defer.inlineCallbacks
+    def consumer(self, routingKey, msg):
+        """
+        Consumer for CaptureBuildStartTime. Gets the build start time.
+        """
+        builder_info = yield self.master.data.get(("builders", msg['builderid']))
+        if self.builder_name == builder_info['name']:
+            ret_val = self.callback(*self.retValParams(msg))
+            context = self.defaultContext(msg)
+            post_data = {
+                self._time_type: ret_val
+            }
+            series_name = self.builder_name + "-build-times"
+            for svc in self.parent_svcs:
+                yield threads.deferToThread(svc.postStatsValue, post_data, series_name,
+                                            context)
+
+        else:
+            yield defer.returnValue(None)
+
+
+class CaptureBuildStartTime(CaptureBuildTimes):
+    """
+    Capture methods for capturing build start times.
+    """
+    def __init__(self, builder_name, callback=None):
+        def default_callback(start_time):
+            return start_time.isoformat()
+        if not callback:
+            callback = default_callback
+        self._time_type = "start-time"
+        CaptureBuildTimes.__init__(self, builder_name, callback)
+
+    def retValParams(self, msg):
+        return [msg['started_at']]
+
+
+class CaptureBuildEndTime(CaptureBuildTimes):
+    """
+    Capture methods for capturing build start times.
+    """
+    def __init__(self, builder_name, callback=None):
+        def default_callback(end_time):
+            return end_time.isoformat()
+        if not callback:
+            callback = default_callback
+        self._time_type = "end-time"
+        CaptureBuildTimes.__init__(self, builder_name, callback)
+
+    def retValParams(self, msg):
+        return [msg['complete_at']]
+
+
+class CaptureBuildDuration(CaptureBuildTimes):
+    """
+    Capture methods for capturing build start times.
+    """
+    def __init__(self, builder_name, report_in='seconds', callback=None):
+        def default_callback(start_time, end_time):
+            divisor = 1
+            # it's a closure
+            if report_in == 'minutes':
+                divisor = 60
+            elif report_in == 'hours':
+                divisor = 60 * 60
+            duration = end_time - start_time
+            # cannot use duration.total_seconds() on Python 2.6
+            duration = ((duration.microseconds + (duration.seconds +
+                                                 duration.days * 24 * 3600) * 1e6) / 1e6)
+            return duration / divisor
+
+        if not callback:
+            callback = default_callback
+        self._time_type = "duration"
+        CaptureBuildTimes.__init__(self, builder_name, callback)
+
+    def retValParams(self, msg):
+        return [msg['started_at'], msg['complete_at']]
+
+
+class CaptureData(Capture):
+    """
+    Capture methods for arbitraty data that may not be stored in the Buildbot database.
+    """
+    def __init__(self, data_name, builder_name, callback=None):
+        self.data_name = data_name
+        self.builder_name = builder_name
+
+        if not callback:
+            callback = lambda x: x
+
+        routingKey = ("stats-yieldMetricsValue", "stats-yield-data")
+        Capture.__init__(self, routingKey, callback)
+
+    @defer.inlineCallbacks
+    def consumer(self, routingKey, msg):
+        build_data = msg['build_data']
+        builder_info = yield self.master.data.get(("builders", build_data['builderid']))
+        if self.builder_name == builder_info['name'] and self.data_name == msg['data_name']:
+            ret_val = self.callback(msg['post_data'])
+            context = self.defaultContext(build_data)
+            post_data = ret_val
+            series_name = self.builder_name + "-" + self.data_name
+            for svc in self.parent_svcs:
+                yield threads.deferToThread(svc.postStatsValue, post_data, series_name,
+                                            context)
+
+        else:
+            yield defer.returnValue(None)

master/buildbot/statistics/stats_service.py

@@ -0,0 +1,90 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.util import service
+from buildbot.statistics.storage_backends import StatsStorageBase
+
+
+class StatsService(service.BuildbotService):
+    """
+    A middleware for passing on statistics data to all storage backends.
+    """
+    def checkConfig(self, storage_backends):
+        for sb in storage_backends:
+            if not isinstance(sb, StatsStorageBase):
+                raise TypeError("Invalid type of stats storage service {0!r}. "
+                                "Should be of type StatsStorageBase, "
+                                "is: {0!r}".format(type(StatsStorageBase)))
+
+    def reconfigService(self, storage_backends):
+        log.msg("Reconfiguring StatsService with config: {0!r}".format(storage_backends))
+
+        self.checkConfig(storage_backends)
+
+        self.registeredStorageServices = []
+        for svc in storage_backends:
+            self.registeredStorageServices.append(svc)
+
+        self.consumers = []
+        self.registerConsumers()
+
+    @defer.inlineCallbacks
+    def registerConsumers(self):
+        self.removeConsumers()  # remove existing consumers and add new ones
+        self.consumers = []
+
+        for svc in self.registeredStorageServices:
+            for cap in svc.captures:
+                cap.parent_svcs.append(svc)
+                cap.master = self.master
+                consumer = yield self.master.mq.startConsuming(cap.consumer, cap.routingKey)
+                self.consumers.append(consumer)
+
+    @defer.inlineCallbacks
+    def stopService(self):
+        yield service.BuildbotService.stopService(self)
+        self.removeConsumers()
+
+    @defer.inlineCallbacks
+    def removeConsumers(self):
+        for consumer in self.consumers:
+            yield consumer.stopConsuming()
+        self.consumers = []
+
+    @defer.inlineCallbacks
+    def yieldMetricsValue(self, data_name, post_data, buildid):
+        """
+        A method to allow posting data that is not generated and stored as build-data in
+        the database. This method generates the `stats-yield-data` event to the mq layer
+        which is then consumed in self.postData.
+
+        @params
+        data_name: (str) The unique name for identifying this data.
+        post_data: (dict) A dictionary of key-value pairs that'll be sent for storage.
+        buildid: The buildid of the current Build.
+        """
+        build_data = yield self.master.data.get(('builds', buildid))
+        routingKey = ("stats-yieldMetricsValue", "stats-yield-data")
+
+        msg = dict()
+        msg['data_name'] = data_name
+        msg['post_data'] = post_data
+        msg['build_data'] = build_data
+
+        self.master.mq.produce(routingKey, msg)
+        yield defer.returnValue(None)