diff --git a/example_pymetrics.py b/example_pymetrics.py index b0da8e2..1e380f4 100644 --- a/example_pymetrics.py +++ b/example_pymetrics.py @@ -13,7 +13,6 @@ from pyformance import global_registry - class Collector(object): # TODO: use meters and histograms instead of gauges if possible @@ -23,7 +22,7 @@ def __init__(self, registry=None): registry = global_registry() self.registry = registry self._memory_usage = 0 - + def collect_memory(self): if resource: usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss @@ -34,22 +33,22 @@ def collect_memory(self): self._memory_usage = usage self.registry.gauge("python.memory.usage").set_value(usage) self.registry.gauge("python.memory.increase").set_value(increase) - + def collect_threads(self): counter = 0 alive = 0 daemon = 0 for thread in threading.enumerate(): - counter +=1 + counter += 1 if thread.isDaemon(): daemon += 1 if thread.isAlive(): alive += 1 - #switch_interval = sys.getcheckinterval() + # switch_interval = sys.getcheckinterval() self.registry.gauge("python.thread.count").set_value(counter) self.registry.gauge("python.thread.daemon").set_value(daemon) self.registry.gauge("python.thread.alive").set_value(alive) - + def collect_garbage(self): (count0, count1, count2) = gc.get_count() (threshold0, threshold1, threshold2) = gc.get_threshold() @@ -62,7 +61,7 @@ def collect_garbage(self): self.registry.gauge("python.gc.objects.count").set_value(object_count) self.registry.gauge("python.gc.referrers.count").set_value(referrers_count) self.registry.gauge("python.gc.referents.count").set_value(referents_count) - + def collect_processes(self): counter = 0 alive = 0 @@ -76,18 +75,17 @@ def collect_processes(self): self.registry.gauge("python.processes.count").set_value(counter) self.registry.gauge("python.processes.alive").set_value(alive) self.registry.gauge("python.processes.daemon").set_value(daemon) - - + def collect(self): self.collect_memory() self.collect_garbage() self.collect_threads() self.collect_processes() - - + if __name__ == "__main__": from pyformance.reporters import ConsoleReporter + reporter = ConsoleReporter() col = Collector() col.collect() diff --git a/example_sysmetrics.py b/example_sysmetrics.py index 34ade35..b033025 100644 --- a/example_sysmetrics.py +++ b/example_sysmetrics.py @@ -25,13 +25,15 @@ def collect_disk_io(self, whitelist=[]): if not whitelist or entry in whitelist: for k, v in six.iteritems(stat._asdict()): self.registry.gauge("disk-%s.%s" % (entry, k)).set_value(v) - + def collect_network_io(self, whitelist=[]): stats = psutil.net_io_counters(pernic=True) for entry, stat in six.iteritems(stats): if not whitelist or entry in whitelist: for k, v in six.iteritems(stat._asdict()): - self.registry.gauge("nic-%s.%s" % (entry.replace(" ", "_"), k)).set_value(v) + self.registry.gauge( + "nic-%s.%s" % (entry.replace(" ", "_"), k) + ).set_value(v) def collect_cpu_times(self, whitelist=[]): stats = psutil.cpu_times(percpu=True) @@ -44,23 +46,28 @@ def collect_swap_usage(self): stats = psutil.swap_memory() for k, v in six.iteritems(stats._asdict()): self.registry.gauge("swap.%s" % k).set_value(v) - + def collect_virtmem_usage(self): stats = psutil.virtual_memory() for k, v in six.iteritems(stats._asdict()): - self.registry.gauge("virtmem.%s" % k).set_value(v) - + self.registry.gauge("virtmem.%s" % k).set_value(v) + def collect_uptime(self): uptime = int(time.time()) - int(psutil.boot_time()) self.registry.gauge("uptime").set_value(uptime) def collect_disk_usage(self, whitelist=[]): for partition in psutil.disk_partitions(): - if not whitelist or partition.mountpoint in whitelist or partition.device in whitelist: + if ( + not whitelist + or partition.mountpoint in whitelist + or partition.device in whitelist + ): usage = psutil.disk_usage(partition.mountpoint) if platform.system() == "Windows": - disk_name = "-" + \ - partition.mountpoint.replace("\\", "").replace(":", "") + disk_name = "-" + partition.mountpoint.replace("\\", "").replace( + ":", "" + ) else: disk_name = partition.mountpoint.replace("/", "-") if disk_name == "-": @@ -71,9 +78,9 @@ def collect_disk_usage(self, whitelist=[]): def collect_loadavgs(self): loadavgs = os.getloadavg() - self.registry.gauge('loadavg_1min').set_value(loadavgs[0]) - self.registry.gauge('loadavg_5min').set_value(loadavgs[1]) - self.registry.gauge('loadavg_15min').set_value(loadavgs[2]) + self.registry.gauge("loadavg_1min").set_value(loadavgs[0]) + self.registry.gauge("loadavg_5min").set_value(loadavgs[1]) + self.registry.gauge("loadavg_15min").set_value(loadavgs[2]) def collect(self): self.collect_disk_io() @@ -85,9 +92,10 @@ def collect(self): self.collect_disk_usage() self.collect_loadavgs() - + if __name__ == "__main__": from pyformance.reporters import ConsoleReporter + reporter = ConsoleReporter() col = Collector() col.collect() diff --git a/pyformance/__init__.py b/pyformance/__init__.py index b90ea29..7f48882 100644 --- a/pyformance/__init__.py +++ b/pyformance/__init__.py @@ -1,6 +1,13 @@ -__import__('pkg_resources').declare_namespace(__name__) +__import__("pkg_resources").declare_namespace(__name__) from .registry import MetricsRegistry, global_registry, set_global_registry from .registry import timer, counter, meter, histogram, gauge -from .registry import dump_metrics, clear, count_calls, meter_calls, hist_calls, time_calls +from .registry import ( + dump_metrics, + clear, + count_calls, + meter_calls, + hist_calls, + time_calls, +) from .meters.timer import call_too_long diff --git a/pyformance/meters/histogram.py b/pyformance/meters/histogram.py index d9212ac..9f7b93a 100644 --- a/pyformance/meters/histogram.py +++ b/pyformance/meters/histogram.py @@ -1,7 +1,7 @@ import time import math from threading import Lock -from ..stats. samples import ExpDecayingSample, DEFAULT_SIZE, DEFAULT_ALPHA +from ..stats.samples import ExpDecayingSample, DEFAULT_SIZE, DEFAULT_ALPHA class Histogram(object): @@ -10,8 +10,7 @@ class Histogram(object): A metric which calculates the distribution of a value. """ - def __init__(self, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time, - sample=None): + def __init__(self, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time, sample=None): """ Creates a new instance of a L{Histogram}. """ diff --git a/pyformance/meters/timer.py b/pyformance/meters/timer.py index 27d1721..5d5aa57 100644 --- a/pyformance/meters/timer.py +++ b/pyformance/meters/timer.py @@ -1,4 +1,5 @@ import time + try: from blinker import Namespace except ImportError: @@ -21,8 +22,15 @@ class Timer(object): """ - def __init__(self, threshold=None, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, - clock=time, sink=None, sample=None): + def __init__( + self, + threshold=None, + size=DEFAULT_SIZE, + alpha=DEFAULT_ALPHA, + clock=time, + sink=None, + sample=None, + ): super(Timer, self).__init__() self.meter = Meter(clock=clock) self.hist = Histogram(size=size, alpha=alpha, clock=clock, sample=sample) @@ -99,7 +107,6 @@ def clear(self): class TimerContext(object): - def __init__(self, timer, clock, *args, **kwargs): super(TimerContext, self).__init__() self.clock = clock @@ -111,9 +118,12 @@ def __init__(self, timer, clock, *args, **kwargs): def stop(self): elapsed = self.clock.time() - self.start_time self.timer._update(elapsed) - if self.timer.threshold and self.timer.threshold < elapsed and call_too_long is not None: - call_too_long.send( - self.timer, elapsed=elapsed, *self.args, **self.kwargs) + if ( + self.timer.threshold + and self.timer.threshold < elapsed + and call_too_long is not None + ): + call_too_long.send(self.timer, elapsed=elapsed, *self.args, **self.kwargs) return elapsed def __enter__(self): diff --git a/pyformance/registry.py b/pyformance/registry.py index 198f252..5f6d24a 100644 --- a/pyformance/registry.py +++ b/pyformance/registry.py @@ -13,7 +13,8 @@ class MetricsRegistry(object): a reference back to its service. The service would create a L{MetricsRegistry} to manage all of its metrics tools. """ - def __init__(self, clock = time): + + def __init__(self, clock=time): """ Creates a new L{MetricsRegistry} instance. """ @@ -35,11 +36,11 @@ def add(self, key, metric): :param metric: instance of Histogram, Meter, Gauge, Timer or Counter """ class_map = ( - (Histogram, self._histograms), - (Meter, self._meters), - (Gauge, self._gauges), - (Timer, self._timers), - (Counter, self._counters), + (Histogram, self._histograms), + (Meter, self._meters), + (Gauge, self._gauges), + (Timer, self._timers), + (Counter, self._counters), ) for cls, registry in class_map: if isinstance(metric, cls): @@ -79,7 +80,8 @@ def gauge(self, key, gauge=None, default=float("nan")): if key not in self._gauges: if gauge is None: gauge = SimpleGauge( - default) # raise TypeError("gauge required for registering") + default + ) # raise TypeError("gauge required for registering") elif not isinstance(gauge, Gauge): if not callable(gauge): raise TypeError("gauge getter not callable") @@ -139,26 +141,30 @@ def _get_histogram_metrics(self, key): if key in self._histograms: histogram = self._histograms[key] snapshot = histogram.get_snapshot() - res = {"avg": snapshot.get_mean(), - "count": histogram.get_count(), - "max": snapshot.get_max(), - "min": snapshot.get_min(), - "std_dev": snapshot.get_stddev(), - "75_percentile": snapshot.get_75th_percentile(), - "95_percentile": snapshot.get_95th_percentile(), - "99_percentile": snapshot.get_99th_percentile(), - "999_percentile": snapshot.get_999th_percentile()} + res = { + "avg": snapshot.get_mean(), + "count": histogram.get_count(), + "max": snapshot.get_max(), + "min": snapshot.get_min(), + "std_dev": snapshot.get_stddev(), + "75_percentile": snapshot.get_75th_percentile(), + "95_percentile": snapshot.get_95th_percentile(), + "99_percentile": snapshot.get_99th_percentile(), + "999_percentile": snapshot.get_999th_percentile(), + } return res return {} def _get_meter_metrics(self, key): if key in self._meters: meter = self._meters[key] - res = {"count": meter.get_count(), - "15m_rate": meter.get_fifteen_minute_rate(), - "5m_rate": meter.get_five_minute_rate(), - "1m_rate": meter.get_one_minute_rate(), - "mean_rate": meter.get_mean_rate()} + res = { + "count": meter.get_count(), + "15m_rate": meter.get_fifteen_minute_rate(), + "5m_rate": meter.get_five_minute_rate(), + "1m_rate": meter.get_one_minute_rate(), + "mean_rate": meter.get_mean_rate(), + } return res return {} @@ -166,21 +172,23 @@ def _get_timer_metrics(self, key): if key in self._timers: timer = self._timers[key] snapshot = timer.get_snapshot() - res = {"avg": timer.get_mean(), - "sum": timer.get_sum(), - "count": timer.get_count(), - "max": timer.get_max(), - "min": timer.get_min(), - "std_dev": timer.get_stddev(), - "15m_rate": timer.get_fifteen_minute_rate(), - "5m_rate": timer.get_five_minute_rate(), - "1m_rate": timer.get_one_minute_rate(), - "mean_rate": timer.get_mean_rate(), - "50_percentile": snapshot.get_median(), - "75_percentile": snapshot.get_75th_percentile(), - "95_percentile": snapshot.get_95th_percentile(), - "99_percentile": snapshot.get_99th_percentile(), - "999_percentile": snapshot.get_999th_percentile()} + res = { + "avg": timer.get_mean(), + "sum": timer.get_sum(), + "count": timer.get_count(), + "max": timer.get_max(), + "min": timer.get_min(), + "std_dev": timer.get_stddev(), + "15m_rate": timer.get_fifteen_minute_rate(), + "5m_rate": timer.get_five_minute_rate(), + "1m_rate": timer.get_one_minute_rate(), + "mean_rate": timer.get_mean_rate(), + "50_percentile": snapshot.get_median(), + "75_percentile": snapshot.get_75th_percentile(), + "95_percentile": snapshot.get_95th_percentile(), + "99_percentile": snapshot.get_99th_percentile(), + "999_percentile": snapshot.get_999th_percentile(), + } return res return {} @@ -194,9 +202,13 @@ def get_metrics(self, key): :return: C{dict} """ metrics = {} - for getter in (self._get_counter_metrics, self._get_histogram_metrics, - self._get_meter_metrics, self._get_timer_metrics, - self._get_gauge_metrics): + for getter in ( + self._get_counter_metrics, + self._get_histogram_metrics, + self._get_meter_metrics, + self._get_timer_metrics, + self._get_gauge_metrics, + ): metrics.update(getter(key)) return metrics @@ -207,11 +219,13 @@ def dump_metrics(self): :return: C{list} of C{dict} of metrics """ metrics = {} - for metric_type in (self._counters, - self._histograms, - self._meters, - self._timers, - self._gauges): + for metric_type in ( + self._counters, + self._histograms, + self._meters, + self._timers, + self._gauges, + ): for key in metric_type.keys(): metrics[key] = self.get_metrics(key) @@ -229,16 +243,17 @@ class RegexRegistry(MetricsRegistry): /api/users/1/edit -> users/edit /api/users/2/edit -> users/edit """ - def __init__(self, pattern = None, clock = time): + + def __init__(self, pattern=None, clock=time): super(RegexRegistry, self).__init__(clock) if pattern is not None: self.pattern = re.compile(pattern) else: - self.pattern = re.compile('^$') + self.pattern = re.compile("^$") def _get_key(self, key): matches = self.pattern.finditer(key) - key = '/'.join((v for match in matches for v in match.groups() if v)) + key = "/".join((v for match in matches for v in match.groups() if v)) return key def timer(self, key): @@ -296,11 +311,13 @@ def dump_metrics(): def clear(): return _global_registry.clear() + def get_qualname(obj): if sys.version_info[0] > 2: return obj.__qualname__ return obj.__name__ + def count_calls(fn): """ Decorator to track the number of times a function is called. @@ -311,10 +328,12 @@ def count_calls(fn): :return: the decorated function :rtype: C{func} """ + @functools.wraps(fn) def wrapper(*args, **kwargs): counter("%s_calls" % get_qualname(fn)).inc() return fn(*args, **kwargs) + return wrapper @@ -328,10 +347,12 @@ def meter_calls(fn): :return: the decorated function :rtype: C{func} """ + @functools.wraps(fn) def wrapper(*args, **kwargs): meter("%s_calls" % get_qualname(fn)).mark() return fn(*args, **kwargs) + return wrapper @@ -345,6 +366,7 @@ def hist_calls(fn): :return: the decorated function :rtype: C{func} """ + @functools.wraps(fn) def wrapper(*args, **kwargs): _histogram = histogram("%s_calls" % get_qualname(fn)) @@ -352,6 +374,7 @@ def wrapper(*args, **kwargs): if type(rtn) in (int, float): _histogram.update(rtn) return rtn + return wrapper @@ -365,10 +388,11 @@ def time_calls(fn): :return: the decorated function :rtype: C{func} """ + @functools.wraps(fn) def wrapper(*args, **kwargs): _timer = timer("%s_calls" % get_qualname(fn)) - with _timer.time(fn = get_qualname(fn)): + with _timer.time(fn=get_qualname(fn)): return fn(*args, **kwargs) - return wrapper + return wrapper diff --git a/pyformance/reporters/__init__.py b/pyformance/reporters/__init__.py index fcced46..279dc6b 100644 --- a/pyformance/reporters/__init__.py +++ b/pyformance/reporters/__init__.py @@ -5,39 +5,47 @@ def HostedGraphiteReporter(*args, **kwargs): from .hosted_graphite_reporter import HostedGraphiteReporter as cls + return cls(*args, **kwargs) def CarbonReporter(*args, **kwargs): from .carbon_reporter import CarbonReporter as cls + return cls(*args, **kwargs) def UdpCarbonReporter(*args, **kwargs): from .carbon_reporter import UdpCarbonReporter as cls + return cls(*args, **kwargs) def ConsoleReporter(*args, **kwargs): from .console_reporter import ConsoleReporter as cls + return cls(*args, **kwargs) def CsvReporter(*args, **kwargs): from .csv_reporter import CsvReporter as cls + return cls(*args, **kwargs) def NewRelicReporter(*args, **kwargs): from .newrelic_reporter import NewRelicReporter as cls + return cls(*args, **kwargs) def InfluxReporter(*args, **kwargs): from .influx import InfluxReporter as cls + return cls(*args, **kwargs) def OpenTSDBReporter(*args, **kwargs): from .opentsdb_reporter import OpenTSDBReporter as cls + return cls(*args, **kwargs) diff --git a/pyformance/reporters/carbon_reporter.py b/pyformance/reporters/carbon_reporter.py index 9231b55..ad5402b 100644 --- a/pyformance/reporters/carbon_reporter.py +++ b/pyformance/reporters/carbon_reporter.py @@ -8,7 +8,7 @@ from .reporter import Reporter -DEFAULT_CARBON_SERVER = '0.0.0.0' +DEFAULT_CARBON_SERVER = "0.0.0.0" DEFAULT_CARBON_PORT = 2003 @@ -18,9 +18,17 @@ class CarbonReporter(Reporter): Carbon is the network daemon to collect metrics for Graphite """ - def __init__(self, registry=None, reporting_interval=5, prefix="", - server=DEFAULT_CARBON_SERVER, port=DEFAULT_CARBON_PORT, socket_factory=socket.socket, - clock=None, pickle_protocol=False): + def __init__( + self, + registry=None, + reporting_interval=5, + prefix="", + server=DEFAULT_CARBON_SERVER, + port=DEFAULT_CARBON_PORT, + socket_factory=socket.socket, + clock=None, + pickle_protocol=False, + ): super(CarbonReporter, self).__init__(registry, reporting_interval, clock) self.prefix = prefix self.server = server @@ -31,7 +39,7 @@ def __init__(self, registry=None, reporting_interval=5, prefix="", def report_now(self, registry=None, timestamp=None): metrics = self._collect_metrics(registry or self.registry, timestamp) if metrics: - # TODO: keep connection open + # TODO: keep connection open with contextlib.closing(self.socket_factory()) as sock: sock.connect((self.server, self.port)) sock.sendall(metrics) @@ -42,11 +50,14 @@ def _collect_metrics(self, registry, timestamp=None): if self.pickle_protocol: payload = pickle.dumps( [ - ("%s%s.%s" % (self.prefix, metric_name, metric_key), (timestamp, metric_value)) + ( + "%s%s.%s" % (self.prefix, metric_name, metric_key), + (timestamp, metric_value), + ) for metric_name, metric in iteritems(metrics) for metric_key, metric_value in iteritems(metric) ], - protocol=2 + protocol=2, ) header = struct.pack("!L", len(payload)) return header + payload @@ -55,23 +66,30 @@ def _collect_metrics(self, registry, timestamp=None): for metric_name, metric in iteritems(metrics): for metric_key, metric_value in iteritems(metric): metric_line = "%s%s.%s %s %s\n" % ( - self.prefix, metric_name, metric_key, metric_value, timestamp) + self.prefix, + metric_name, + metric_key, + metric_value, + timestamp, + ) metrics_data.append(metric_line) - result = ''.join(metrics_data) + result = "".join(metrics_data) if sys.version_info[0] > 2: return result.encode() return result - + class UdpCarbonReporter(CarbonReporter): - + """ The default CarbonReporter uses TCP. This sub-class uses UDP instead which might be unreliable but it is faster """ - + def report_now(self, registry=None, timestamp=None): metrics = self._collect_metrics(registry or self.registry, timestamp) if metrics: - with contextlib.closing(self.socket_factory(socket.AF_INET, socket.SOCK_DGRAM)) as sock: + with contextlib.closing( + self.socket_factory(socket.AF_INET, socket.SOCK_DGRAM) + ) as sock: sock.sendto(metrics, (self.server, self.port)) diff --git a/pyformance/reporters/console_reporter.py b/pyformance/reporters/console_reporter.py index 2d18727..3146b7f 100644 --- a/pyformance/reporters/console_reporter.py +++ b/pyformance/reporters/console_reporter.py @@ -12,9 +12,10 @@ class ConsoleReporter(Reporter): This is useful for debugging if you want to read the current state on the console. """ - def __init__(self, registry=None, reporting_interval=30, stream=sys.stderr, clock=None): - super(ConsoleReporter, self).__init__( - registry, reporting_interval, clock) + def __init__( + self, registry=None, reporting_interval=30, stream=sys.stderr, clock=None + ): + super(ConsoleReporter, self).__init__(registry, reporting_interval, clock) self.stream = stream def report_now(self, registry=None, timestamp=None): @@ -24,16 +25,16 @@ def report_now(self, registry=None, timestamp=None): def _collect_metrics(self, registry, timestamp=None): timestamp = timestamp or int(round(self.clock.time())) - dt = datetime.datetime(1970, 1, 1) + \ - datetime.timedelta(seconds=timestamp) + dt = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=timestamp) metrics = registry.dump_metrics() - metrics_data = ["== %s ===================================" % - dt.strftime("%Y-%m-%d %H:%M:%S")] + metrics_data = [ + "== %s ===================================" + % dt.strftime("%Y-%m-%d %H:%M:%S") + ] for key in metrics.keys(): values = metrics[key] metrics_data.append("%s:" % key) for value_key in values.keys(): - metrics_data.append( - "%20s = %s" % (value_key, values[value_key])) + metrics_data.append("%20s = %s" % (value_key, values[value_key])) metrics_data.append("") return metrics_data diff --git a/pyformance/reporters/csv_reporter.py b/pyformance/reporters/csv_reporter.py index f538b87..a54bde0 100644 --- a/pyformance/reporters/csv_reporter.py +++ b/pyformance/reporters/csv_reporter.py @@ -12,9 +12,15 @@ class CsvReporter(Reporter): Each metrics gets its own file """ - def __init__(self, registry=None, reporting_interval=30, path=None, separator="\t", clock=None): - super(CsvReporter, self).__init__( - registry, reporting_interval, clock) + def __init__( + self, + registry=None, + reporting_interval=30, + path=None, + separator="\t", + clock=None, + ): + super(CsvReporter, self).__init__(registry, reporting_interval, clock) self.path = path or os.getcwd() if not os.path.exists(self.path): os.makedirs(self.path) @@ -26,8 +32,7 @@ def report_now(self, registry=None, timestamp=None): def _save_metrics(self, registry, timestamp=None): timestamp = timestamp or int(round(self.clock.time())) - dt = datetime.datetime(1970, 1, 1) + \ - datetime.timedelta(seconds=timestamp) + dt = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=timestamp) date = dt.strftime("%Y-%m-%d %H:%M:%S") metrics = registry.dump_metrics() for key in metrics.keys(): @@ -54,4 +59,3 @@ def __enter__(self): def __exit__(self, type, value, traceback): for f in self.files.values(): f.close() - diff --git a/pyformance/reporters/hosted_graphite_reporter.py b/pyformance/reporters/hosted_graphite_reporter.py index 97e5fdd..dbfbc0a 100644 --- a/pyformance/reporters/hosted_graphite_reporter.py +++ b/pyformance/reporters/hosted_graphite_reporter.py @@ -16,10 +16,16 @@ class HostedGraphiteReporter(Reporter): """ def __init__( - self, hosted_graphite_api_key, registry=None, reporting_interval=10, url="https://hostedgraphite.com/api/v1/sink", - clock=None): + self, + hosted_graphite_api_key, + registry=None, + reporting_interval=10, + url="https://hostedgraphite.com/api/v1/sink", + clock=None, + ): super(HostedGraphiteReporter, self).__init__( - registry, reporting_interval, clock) + registry, reporting_interval, clock + ) self.url = url self.api_key = hosted_graphite_api_key @@ -29,8 +35,10 @@ def report_now(self, registry=None, timestamp=None): try: # XXX: better use http-keepalive/pipelining somehow? request = urllib2.Request(self.url, metrics) - request.add_header("Authorization", "Basic %s" % - base64.encodestring(self.api_key).strip()) + request.add_header( + "Authorization", + "Basic %s" % base64.encodestring(self.api_key).strip(), + ) result = urllib2.urlopen(request) except Exception as e: print(e, file=sys.stderr) @@ -42,6 +50,10 @@ def _collect_metrics(self, registry, timestamp=None): for key in metrics.keys(): for value_key in metrics[key].keys(): metric_line = "%s.%s %s %s\n" % ( - key, value_key, metrics[key][value_key], timestamp) + key, + value_key, + metrics[key][value_key], + timestamp, + ) metrics_data.append(metric_line) - return ''.join(metrics_data) + return "".join(metrics_data) diff --git a/pyformance/reporters/influx.py b/pyformance/reporters/influx.py index 404d29c..2b74170 100644 --- a/pyformance/reporters/influx.py +++ b/pyformance/reporters/influx.py @@ -2,6 +2,7 @@ import base64 import logging + try: from urllib2 import quote, urlopen, Request, URLError except ImportError: @@ -13,13 +14,14 @@ LOG = logging.getLogger(__name__) -DEFAULT_INFLUX_SERVER = '127.0.0.1' +DEFAULT_INFLUX_SERVER = "127.0.0.1" DEFAULT_INFLUX_PORT = 8086 DEFAULT_INFLUX_DATABASE = "metrics" DEFAULT_INFLUX_USERNAME = None DEFAULT_INFLUX_PASSWORD = None DEFAULT_INFLUX_PROTOCOL = "http" + class InfluxReporter(Reporter): """ @@ -27,14 +29,21 @@ class InfluxReporter(Reporter): (based on https://influxdb.com/docs/v1.1/guides/writing_data.html) """ - def __init__(self, registry=None, reporting_interval=5, prefix="", - database=DEFAULT_INFLUX_DATABASE, server=DEFAULT_INFLUX_SERVER, - username=DEFAULT_INFLUX_USERNAME, - password=DEFAULT_INFLUX_PASSWORD, - port=DEFAULT_INFLUX_PORT, protocol=DEFAULT_INFLUX_PROTOCOL, - autocreate_database=False, clock=None): - super(InfluxReporter, self).__init__( - registry, reporting_interval, clock) + def __init__( + self, + registry=None, + reporting_interval=5, + prefix="", + database=DEFAULT_INFLUX_DATABASE, + server=DEFAULT_INFLUX_SERVER, + username=DEFAULT_INFLUX_USERNAME, + password=DEFAULT_INFLUX_PASSWORD, + port=DEFAULT_INFLUX_PORT, + protocol=DEFAULT_INFLUX_PROTOCOL, + autocreate_database=False, + clock=None, + ): + super(InfluxReporter, self).__init__(registry, reporting_interval, clock) self.prefix = prefix self.database = database self.username = username @@ -51,15 +60,19 @@ def _create_database(self): request = Request(url + "?q=" + q) if self.username: auth = _encode_username(self.username, self.password) - request.add_header("Authorization", "Basic %s" % auth.decode('utf-8')) + request.add_header("Authorization", "Basic %s" % auth.decode("utf-8")) try: response = urlopen(request) _result = response.read() # Only set if we actually were able to get a successful response self._did_create_database = True except URLError as err: - LOG.warning("Cannot create database %s to %s: %s", - self.database, self.server, err.reason) + LOG.warning( + "Cannot create database %s to %s: %s", + self.database, + self.server, + err.reason, + ) def report_now(self, registry=None, timestamp=None): if self.autocreate_database and not self._did_create_database: @@ -72,9 +85,12 @@ def report_now(self, registry=None, timestamp=None): table = key else: table = "%s.%s" % (self.prefix, key) - values = ",".join(["%s=%s" % (k, v if type(v) is not str \ - else '"{}"'.format(v)) - for (k, v) in metric_values.items()]) + values = ",".join( + [ + "%s=%s" % (k, v if type(v) is not str else '"{}"'.format(v)) + for (k, v) in metric_values.items() + ] + ) line = "%s %s %s" % (table, values, timestamp) post_data.append(line) post_data = "\n".join(post_data) @@ -83,15 +99,14 @@ def report_now(self, registry=None, timestamp=None): request = Request(url, post_data.encode("utf-8")) if self.username: auth = _encode_username(self.username, self.password) - request.add_header("Authorization", "Basic %s" % auth.decode('utf-8')) + request.add_header("Authorization", "Basic %s" % auth.decode("utf-8")) try: response = urlopen(request) response.read() except URLError as err: - LOG.warning("Cannot write to %s: %s", - self.server, err.reason) + LOG.warning("Cannot write to %s: %s", self.server, err.reason) def _encode_username(username, password): - auth_string = ('%s:%s' % (username, password)).encode() + auth_string = ("%s:%s" % (username, password)).encode() return base64.b64encode(auth_string) diff --git a/pyformance/reporters/newrelic_reporter.py b/pyformance/reporters/newrelic_reporter.py index 2203400..824c401 100644 --- a/pyformance/reporters/newrelic_reporter.py +++ b/pyformance/reporters/newrelic_reporter.py @@ -17,12 +17,11 @@ from .reporter import Reporter -DEFAULT_CARBON_SERVER = '0.0.0.0' +DEFAULT_CARBON_SERVER = "0.0.0.0" DEFAULT_CARBON_PORT = 2003 class NewRelicSink(object): - def __init__(self): self.total = 0 self.count = 0 @@ -53,26 +52,36 @@ class NewRelicReporter(Reporter): """ MAX_METRICS_PER_REQUEST = 10000 - PLATFORM_URL = 'https://platform-api.newrelic.com/platform/v1/metrics' - - def __init__(self, license_key, registry=None, name=socket.gethostname(), reporting_interval=5, prefix="", - clock=None): - super(NewRelicReporter, self).__init__( - registry, reporting_interval, clock) + PLATFORM_URL = "https://platform-api.newrelic.com/platform/v1/metrics" + + def __init__( + self, + license_key, + registry=None, + name=socket.gethostname(), + reporting_interval=5, + prefix="", + clock=None, + ): + super(NewRelicReporter, self).__init__(registry, reporting_interval, clock) self.name = name self.prefix = prefix - self.http_headers = {'Accept': 'application/json', - 'Content-Type': 'application/json', - 'X-License-Key': license_key} - + self.http_headers = { + "Accept": "application/json", + "Content-Type": "application/json", + "X-License-Key": license_key, + } def report_now(self, registry=None, timestamp=None): metrics = self.collect_metrics(registry or self.registry) if metrics: try: # XXX: better use http-keepalive/pipelining somehow? - request = urllib.Request(self.PLATFORM_URL, metrics.encode() if sys.version_info[0] > 2 else metrics) + request = urllib.Request( + self.PLATFORM_URL, + metrics.encode() if sys.version_info[0] > 2 else metrics, + ) for k, v in self.http_headers.items(): request.add_header(k, v) result = urllib.urlopen(request) @@ -88,9 +97,11 @@ def agent_data(self): :rtype: dict """ - return {'host': socket.gethostname(), - 'pid': os.getpid(), - 'version': __version__} + return { + "host": socket.gethostname(), + "pid": os.getpid(), + "version": __version__, + } def create_metrics(self, registry): results = {} @@ -102,26 +113,28 @@ def create_metrics(self, registry): if not sink.count: continue - full_key = 'Component/%s%s' % (self.prefix, key) - results[full_key.replace('.', '/')] = { + full_key = "Component/%s%s" % (self.prefix, key) + results[full_key.replace(".", "/")] = { "total": sink.total, "count": sink.count, "min": sink.min, "max": sink.max, - "sum_of_squares": sink.sum_of_squares + "sum_of_squares": sink.sum_of_squares, } sink.__init__() return results def collect_metrics(self, registry): body = { - 'agent': self.agent_data, - 'components': [{ - 'guid': 'com.github.pyformance', - 'name': self.name, - 'duration': self.reporting_interval, - 'metrics': self.create_metrics(registry) - }] + "agent": self.agent_data, + "components": [ + { + "guid": "com.github.pyformance", + "name": self.name, + "duration": self.reporting_interval, + "metrics": self.create_metrics(registry), + } + ], } return json.dumps(body, ensure_ascii=False, sort_keys=True) diff --git a/pyformance/reporters/opentsdb_reporter.py b/pyformance/reporters/opentsdb_reporter.py index 6d39825..e28e525 100644 --- a/pyformance/reporters/opentsdb_reporter.py +++ b/pyformance/reporters/opentsdb_reporter.py @@ -18,11 +18,20 @@ class OpenTSDBReporter(Reporter): This reporter requires a tuple (application_name, write_key) to put data to opentsdb database """ - def __init__(self, application_name, write_key, url, registry=None, reporting_interval=10, clock=None, prefix="", - tags={}): - super(OpenTSDBReporter, self).__init__(registry=registry, - reporting_interval=reporting_interval, - clock=clock) + def __init__( + self, + application_name, + write_key, + url, + registry=None, + reporting_interval=10, + clock=None, + prefix="", + tags={}, + ): + super(OpenTSDBReporter, self).__init__( + registry=registry, reporting_interval=reporting_interval, clock=clock + ) self.url = url self.application_name = application_name self.write_key = write_key @@ -33,11 +42,17 @@ def report_now(self, registry=None, timestamp=None): metrics = self._collect_metrics(registry or self.registry, timestamp) if metrics: try: - request = urllib.Request(self.url, - data=json.dumps(metrics).encode("utf-8"), - headers={'content-type': "application/json"}) - authentication_data = "{0}:{1}".format(self.application_name, self.write_key) - auth_header = base64.b64encode(bytes(authentication_data.encode("utf-8"))).decode("utf-8") + request = urllib.Request( + self.url, + data=json.dumps(metrics).encode("utf-8"), + headers={"content-type": "application/json"}, + ) + authentication_data = "{0}:{1}".format( + self.application_name, self.write_key + ) + auth_header = base64.b64encode( + bytes(authentication_data.encode("utf-8")) + ).decode("utf-8") request.add_header("Authorization", "Basic {0}".format(auth_header)) urllib.urlopen(request) except Exception as e: @@ -49,11 +64,12 @@ def _collect_metrics(self, registry, timestamp=None): metrics_data = [] for key in metrics.keys(): for value_key in metrics[key].keys(): - metrics_data.append({ - 'metric': "{0}{1}.{2}".format(self.prefix, key, value_key), - 'value': metrics[key][value_key], - 'timestamp': timestamp, - 'tags': self.tags, - }) + metrics_data.append( + { + "metric": "{0}{1}.{2}".format(self.prefix, key, value_key), + "value": metrics[key][value_key], + "timestamp": timestamp, + "tags": self.tags, + } + ) return metrics_data - diff --git a/pyformance/reporters/reporter.py b/pyformance/reporters/reporter.py index 947327a..d2a4b62 100644 --- a/pyformance/reporters/reporter.py +++ b/pyformance/reporters/reporter.py @@ -5,10 +5,12 @@ class Reporter(object): - def create_thread(self): # noinspection PyAttributeOutsideInit - self._loop_thread = Thread(target=self._loop, name="pyformance reporter {0}".format(get_qualname(type(self)))) + self._loop_thread = Thread( + target=self._loop, + name="pyformance reporter {0}".format(get_qualname(type(self))), + ) self._loop_thread.setDaemon(True) def __init__(self, registry=None, reporting_interval=30, clock=None): diff --git a/pyformance/reporters/syslog_reporter.py b/pyformance/reporters/syslog_reporter.py index e3abd5e..3311029 100644 --- a/pyformance/reporters/syslog_reporter.py +++ b/pyformance/reporters/syslog_reporter.py @@ -8,7 +8,7 @@ from .reporter import Reporter -DEFAULT_SYSLOG_ADDRESS = '/dev/log' +DEFAULT_SYSLOG_ADDRESS = "/dev/log" DEFAULT_SYSLOG_SOCKTYPE = socket.SOCK_DGRAM DEFAULT_SYSLOG_FACILITY = logging.handlers.SysLogHandler.LOG_USER @@ -19,42 +19,47 @@ class SysLogReporter(Reporter): Syslog is a way for network devices to send event messages to a logging server """ - def __init__(self, registry=None, reporting_interval=5, tag="pyformance", clock=None, - address=DEFAULT_SYSLOG_ADDRESS, socktype=DEFAULT_SYSLOG_SOCKTYPE, - facility=DEFAULT_SYSLOG_FACILITY): + def __init__( + self, + registry=None, + reporting_interval=5, + tag="pyformance", + clock=None, + address=DEFAULT_SYSLOG_ADDRESS, + socktype=DEFAULT_SYSLOG_SOCKTYPE, + facility=DEFAULT_SYSLOG_FACILITY, + ): super(SysLogReporter, self).__init__(registry, reporting_interval, clock) - handler = logging.handlers.SysLogHandler(address=address, facility=facility, socktype=socktype) + handler = logging.handlers.SysLogHandler( + address=address, facility=facility, socktype=socktype + ) handler.append_nul = False - if tag is not None and tag != '': - if sys.version_info >= (3,3): - handler.ident = tag + ': ' + if tag is not None and tag != "": + if sys.version_info >= (3, 3): + handler.ident = tag + ": " else: - formatter = logging.Formatter('{}: %(message)s'.format(tag)) + formatter = logging.Formatter("{}: %(message)s".format(tag)) handler.setFormatter(formatter) - logger = logging.getLogger('pyformance') + logger = logging.getLogger("pyformance") logger.setLevel(logging.INFO) logger.addHandler(handler) self.logger = logger - def report_now(self, registry=None, timestamp=None): metrics = self._collect_metrics(registry or self.registry, timestamp) if metrics: self.logger.info(metrics) - def _collect_metrics(self, registry, timestamp=None): timestamp = timestamp or int(round(self.clock.time())) - metrics_data = {'timestamp': timestamp} + metrics_data = {"timestamp": timestamp} metrics = registry.dump_metrics() for metric_name, metric in iteritems(metrics): for metric_key, metric_value in iteritems(metric): - metrics_data['{}.{}'.format(metric_name, metric_key)] = metric_value + metrics_data["{}.{}".format(metric_name, metric_key)] = metric_value result = json.dumps(metrics_data, sort_keys=True) return result - - diff --git a/pyformance/stats/moving_average.py b/pyformance/stats/moving_average.py index 0fa5707..4ac6eaa 100644 --- a/pyformance/stats/moving_average.py +++ b/pyformance/stats/moving_average.py @@ -7,6 +7,7 @@ class ExpWeightedMovingAvg(object): """ An exponentially-weighted moving average. """ + INTERVAL = 5.0 # seconds SECONDS_PER_MINUTE = 60.0 @@ -51,7 +52,7 @@ def tick(self): self.uncounted = 0.0 if self.rate >= 0: - self.rate += (self._alpha(interval) * (instant_rate - self.rate)) + self.rate += self._alpha(interval) * (instant_rate - self.rate) else: self.rate = instant_rate diff --git a/pyformance/stats/samples.py b/pyformance/stats/samples.py index e9649a7..fa3515d 100644 --- a/pyformance/stats/samples.py +++ b/pyformance/stats/samples.py @@ -49,8 +49,7 @@ def clear(self): self.priorities = [] self.counter = 0 self.start_time = self.clock.time() - self.next_time = self.clock.time() + \ - ExpDecayingSample.RESCALE_THREASHOLD + self.next_time = self.clock.time() + ExpDecayingSample.RESCALE_THREASHOLD def get_size(self): return self.counter if self.counter < self.size else self.size @@ -65,8 +64,7 @@ def update(self, value): if self.size == 0: return self._rescale_if_necessary() - priority = self._weight( - self.clock.time() - self.start_time) / random.random() + priority = self._weight(self.clock.time() - self.start_time) / random.random() new_counter = self.counter + 1 self.counter = new_counter @@ -90,15 +88,13 @@ def _rescale_if_necessary(self): self._rescale() def _rescale(self): - self.next_time = self.clock.time() + \ - ExpDecayingSample.RESCALE_THREASHOLD + self.next_time = self.clock.time() + ExpDecayingSample.RESCALE_THREASHOLD old_start_time = self.start_time self.start_time = self.clock.time() new_values = {} new_priorities = [] for key, val in self.values.items(): - priority = key * \ - math.exp(-self.alpha * (self.start_time - old_start_time)) + priority = key * math.exp(-self.alpha * (self.start_time - old_start_time)) new_values[priority] = val heapq.heappush(new_priorities, priority) self.values = new_values diff --git a/setup.py b/setup.py index cd6bb2c..35c6094 100644 --- a/setup.py +++ b/setup.py @@ -8,24 +8,25 @@ with open(_IN_PACKAGE_DIR("__version__.py")) as version_file: exec(version_file.read()) -install_requires = ['six'] # optional: ["blinker==1.2"] -if platform.python_version() < '2.7': - install_requires.append('unittest2') +install_requires = ["six"] # optional: ["blinker==1.2"] +if platform.python_version() < "2.7": + install_requires.append("unittest2") -setup(name="pyformance", - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.4", - ], - description="Performance metrics, based on Coda Hale's Yammer metrics", - license="Apache 2.0", - author="Omer Getrel", - author_email="omer.gertel@gmail.com", - version=__version__, - packages=find_packages(), - data_files=[], - install_requires=install_requires, - scripts=[], - ) +setup( + name="pyformance", + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.4", + ], + description="Performance metrics, based on Coda Hale's Yammer metrics", + license="Apache 2.0", + author="Omer Getrel", + author_email="omer.gertel@gmail.com", + version=__version__, + packages=find_packages(), + data_files=[], + install_requires=install_requires, + scripts=[], +) diff --git a/tests/__init__.py b/tests/__init__.py index 8d4057d..38de85b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,12 +1,12 @@ import platform -if platform.python_version() < '2.7': + +if platform.python_version() < "2.7": import unittest2 as unittest else: import unittest class ManualClock(object): - def __init__(self): super(ManualClock, self).__init__() self.now = 0 diff --git a/tests/test__carbon_reporter.py b/tests/test__carbon_reporter.py index 8bf7297..ddb2cec 100644 --- a/tests/test__carbon_reporter.py +++ b/tests/test__carbon_reporter.py @@ -4,8 +4,8 @@ from tests import TimedTestCase import pickle -class TestCarbonReporter(TimedTestCase): +class TestCarbonReporter(TimedTestCase): def setUp(self): super(TestCarbonReporter, self).setUp() self.output = BytesIO() @@ -47,94 +47,107 @@ def capture_test_metrics(self): def test_report_now_plain(self): r = CarbonReporter( - registry=self.registry, reporting_interval=1, clock=self.clock, - socket_factory=lambda: self) + registry=self.registry, + reporting_interval=1, + clock=self.clock, + socket_factory=lambda: self, + ) self.capture_test_metrics() r.report_now() test_data = sorted(self.output.getvalue().decode().splitlines()) - expected_data = sorted([ - 'counter-2.count -2 2', - 'c1.count 1 2', - 'gsimple.value 42 2', - 'gcb.value 123 2', - 't1.1m_rate 0 2', - 't1.999_percentile 1 2', - 't1.15m_rate 0 2', - 't1.99_percentile 1 2', - 't1.mean_rate 1.0 2', - 't1.95_percentile 1 2', - 't1.min 1 2', - 't1.50_percentile 1 2', - 't1.5m_rate 0 2', - 't1.count 1.0 2', - 't1.75_percentile 1 2', - 't1.std_dev 0.0 2', - 't1.max 1 2', - 't1.sum 1.0 2', - 't1.avg 1.0 2', - 'hist.count 10.0 2', - 'hist.999_percentile 512 2', - 'hist.99_percentile 512 2', - 'hist.min 1 2', - 'hist.95_percentile 512 2', - 'hist.75_percentile 160.0 2', - 'hist.std_dev 164.94851048466944 2' \ - if PY3 else 'hist.std_dev 164.948510485 2', - 'hist.max 512 2', - 'hist.avg 102.3 2', - 'm1.count 1.0 2', - 'm1.1m_rate 0 2', - 'm1.15m_rate 0 2', - 'm1.5m_rate 0 2', - 'm1.mean_rate 1.0 2', - ]) + expected_data = sorted( + [ + "counter-2.count -2 2", + "c1.count 1 2", + "gsimple.value 42 2", + "gcb.value 123 2", + "t1.1m_rate 0 2", + "t1.999_percentile 1 2", + "t1.15m_rate 0 2", + "t1.99_percentile 1 2", + "t1.mean_rate 1.0 2", + "t1.95_percentile 1 2", + "t1.min 1 2", + "t1.50_percentile 1 2", + "t1.5m_rate 0 2", + "t1.count 1.0 2", + "t1.75_percentile 1 2", + "t1.std_dev 0.0 2", + "t1.max 1 2", + "t1.sum 1.0 2", + "t1.avg 1.0 2", + "hist.count 10.0 2", + "hist.999_percentile 512 2", + "hist.99_percentile 512 2", + "hist.min 1 2", + "hist.95_percentile 512 2", + "hist.75_percentile 160.0 2", + "hist.std_dev 164.94851048466944 2" + if PY3 + else "hist.std_dev 164.948510485 2", + "hist.max 512 2", + "hist.avg 102.3 2", + "m1.count 1.0 2", + "m1.1m_rate 0 2", + "m1.15m_rate 0 2", + "m1.5m_rate 0 2", + "m1.mean_rate 1.0 2", + ] + ) self.assertEqual(test_data, expected_data) def test_report_now_pickle(self): r = CarbonReporter( - registry=self.registry, reporting_interval=1, clock=self.clock, - socket_factory=lambda: self, pickle_protocol=True) + registry=self.registry, + reporting_interval=1, + clock=self.clock, + socket_factory=lambda: self, + pickle_protocol=True, + ) self.capture_test_metrics() r.report_now() test_data = sorted(pickle.loads(self.output.getvalue()[4:])) - expected_data = sorted([ - ('counter-2.count', (2, -2.0)), - ('c1.count', (2, 1)), - ('gsimple.value', (2, 42.0)), - ('gcb.value', (2, 123.0)), - ('t1.1m_rate', (2, 0.0)), - ('t1.999_percentile', (2, 1)), - ('t1.15m_rate', (2, 0.0)), - ('t1.99_percentile', (2, 1)), - ('t1.mean_rate', (2, 1)), - ('t1.95_percentile', (2, 1)), - ('t1.min', (2, 1)), - ('t1.50_percentile', (2, 1)), - ('t1.5m_rate', (2, 0.0)), - ('t1.count', (2, 1)), - ('t1.75_percentile', (2, 1)), - ('t1.std_dev', (2, 0.0)), - ('t1.max', (2, 1)), - ('t1.sum', (2, 1)), - ('t1.avg', (2, 1)), - ('hist.count', (2, 10.0)), - ('hist.999_percentile', (2, 512.0)), - ('hist.99_percentile', (2, 512.0)), - ('hist.min', (2, 1)), - ('hist.95_percentile', (2, 512.0)), - ('hist.75_percentile', (2, 160.0)), - ('hist.std_dev', (2, 164.94851048466944)), - ('hist.max', (2, 512.0)), - ('hist.avg', (2, 102.3)), - ('m1.count', (2, 1)), - ('m1.1m_rate', (2, 0.0)), - ('m1.15m_rate', (2, 0.0)), - ('m1.5m_rate', (2, 0.0)), - ('m1.mean_rate', (2, 1)) - ]) + expected_data = sorted( + [ + ("counter-2.count", (2, -2.0)), + ("c1.count", (2, 1)), + ("gsimple.value", (2, 42.0)), + ("gcb.value", (2, 123.0)), + ("t1.1m_rate", (2, 0.0)), + ("t1.999_percentile", (2, 1)), + ("t1.15m_rate", (2, 0.0)), + ("t1.99_percentile", (2, 1)), + ("t1.mean_rate", (2, 1)), + ("t1.95_percentile", (2, 1)), + ("t1.min", (2, 1)), + ("t1.50_percentile", (2, 1)), + ("t1.5m_rate", (2, 0.0)), + ("t1.count", (2, 1)), + ("t1.75_percentile", (2, 1)), + ("t1.std_dev", (2, 0.0)), + ("t1.max", (2, 1)), + ("t1.sum", (2, 1)), + ("t1.avg", (2, 1)), + ("hist.count", (2, 10.0)), + ("hist.999_percentile", (2, 512.0)), + ("hist.99_percentile", (2, 512.0)), + ("hist.min", (2, 1)), + ("hist.95_percentile", (2, 512.0)), + ("hist.75_percentile", (2, 160.0)), + ("hist.std_dev", (2, 164.94851048466944)), + ("hist.max", (2, 512.0)), + ("hist.avg", (2, 102.3)), + ("m1.count", (2, 1)), + ("m1.1m_rate", (2, 0.0)), + ("m1.15m_rate", (2, 0.0)), + ("m1.5m_rate", (2, 0.0)), + ("m1.mean_rate", (2, 1)), + ] + ) self.assertEqual(test_data, expected_data) if __name__ == "__main__": import unittest + unittest.main() diff --git a/tests/test__console_reporter.py b/tests/test__console_reporter.py index 1edb526..96339a9 100644 --- a/tests/test__console_reporter.py +++ b/tests/test__console_reporter.py @@ -1,4 +1,5 @@ import sys + if sys.version_info[0] < 3: from StringIO import StringIO else: @@ -10,7 +11,6 @@ class TestConsoleReporter(TimedTestCase): - def setUp(self): super(TestConsoleReporter, self).setUp() self.output = StringIO() @@ -24,7 +24,11 @@ def tearDown(self): def test_report_now(self): r = ConsoleReporter( - registry=self.registry, reporting_interval=1, stream=self.output, clock=self.clock) + registry=self.registry, + reporting_interval=1, + stream=self.output, + clock=self.clock, + ) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) @@ -41,41 +45,54 @@ def test_report_now(self): c2.dec() self.clock.add(1) r.report_now() - self.assertEqual(self.output.getvalue().splitlines().sort(), [ - '== 1970-01-01 00:00:01 ===================================', - 'counter-2:', ' count = -2', - 'gsimple:', ' value = 42', - 'gcb:', ' value = 123', - 't1:', ' 1m_rate = 0', - ' 999_percentile = 1', - ' 15m_rate = 0', - ' 99_percentile = 1', - ' mean_rate = 1.0', - ' 95_percentile = 1', - ' min = 1', - ' 5m_rate = 0', - ' count = 1.0', - ' 75_percentile = 1', - ' std_dev = 0.0', - ' max = 1', - ' avg = 1.0', - 'hist:', ' count = 10.0', - ' 999_percentile = 512', - ' 99_percentile = 512', - ' min = 1', - ' 95_percentile = 512', - ' 75_percentile = 160.0', - ' std_dev = 164.94851048466944', - ' max = 512', - ' avg = 102.3', - 'm1:', ' count = 1.0', - ' 1m_rate = 0', - ' 15m_rate = 0', - ' 5m_rate = 0', - ' mean_rate = 1.0', - 'c1:', ' count = 1', ''].sort()) + self.assertEqual( + self.output.getvalue().splitlines().sort(), + [ + "== 1970-01-01 00:00:01 ===================================", + "counter-2:", + " count = -2", + "gsimple:", + " value = 42", + "gcb:", + " value = 123", + "t1:", + " 1m_rate = 0", + " 999_percentile = 1", + " 15m_rate = 0", + " 99_percentile = 1", + " mean_rate = 1.0", + " 95_percentile = 1", + " min = 1", + " 5m_rate = 0", + " count = 1.0", + " 75_percentile = 1", + " std_dev = 0.0", + " max = 1", + " avg = 1.0", + "hist:", + " count = 10.0", + " 999_percentile = 512", + " 99_percentile = 512", + " min = 1", + " 95_percentile = 512", + " 75_percentile = 160.0", + " std_dev = 164.94851048466944", + " max = 512", + " avg = 102.3", + "m1:", + " count = 1.0", + " 1m_rate = 0", + " 15m_rate = 0", + " 5m_rate = 0", + " mean_rate = 1.0", + "c1:", + " count = 1", + "", + ].sort(), + ) if __name__ == "__main__": import unittest + unittest.main() diff --git a/tests/test__counter.py b/tests/test__counter.py index 647cf6e..dcf038a 100644 --- a/tests/test__counter.py +++ b/tests/test__counter.py @@ -3,7 +3,6 @@ class CounterTestCase(TimedTestCase): - def setUp(self): super(CounterTestCase, self).setUp() self.counter = Counter() diff --git a/tests/test__csv_reporter.py b/tests/test__csv_reporter.py index fb977e1..bbe72cc 100644 --- a/tests/test__csv_reporter.py +++ b/tests/test__csv_reporter.py @@ -2,6 +2,7 @@ import os import shutil import tempfile + if sys.version_info[0] < 3: from StringIO import StringIO else: @@ -13,7 +14,6 @@ class TestCsvReporter(TimedTestCase): - def setUp(self): super(TestCsvReporter, self).setUp() self.clock = ManualClock() @@ -31,18 +31,22 @@ def test_report_now(self): g1.set_value(123) with CsvReporter( - registry=self.registry, reporting_interval=1, clock=self.clock, - path=self.path) as r: + registry=self.registry, + reporting_interval=1, + clock=self.clock, + path=self.path, + ) as r: r.report_now() output_filename = os.path.join(self.path, "gauge1.csv") output = open(output_filename).read() - self.assertEqual(output.splitlines(), [ - 'timestamp\tvalue', '1970-01-01 00:00:00\t123' - ]) + self.assertEqual( + output.splitlines(), ["timestamp\tvalue", "1970-01-01 00:00:00\t123"] + ) if __name__ == "__main__": import unittest + unittest.main() diff --git a/tests/test__gauge.py b/tests/test__gauge.py index 376e386..4876a83 100644 --- a/tests/test__gauge.py +++ b/tests/test__gauge.py @@ -3,7 +3,6 @@ class CallbackGaugeTestCase(TimedTestCase): - def setUp(self): super(CallbackGaugeTestCase, self).setUp() self._value = None diff --git a/tests/test__histogram.py b/tests/test__histogram.py index 971e586..24964de 100644 --- a/tests/test__histogram.py +++ b/tests/test__histogram.py @@ -3,7 +3,6 @@ class HistogramTestCase(TimedTestCase): - def test__a_sample_of_100_from_1000(self): hist = Histogram(100, 0.99) for i in range(1000): diff --git a/tests/test__influx_reporter.py b/tests/test__influx_reporter.py index 12ce1eb..cafa338 100644 --- a/tests/test__influx_reporter.py +++ b/tests/test__influx_reporter.py @@ -10,6 +10,7 @@ from pyformance import MetricsRegistry from tests import TimedTestCase + class TestInfluxReporter(TimedTestCase): def setUp(self): super(TestInfluxReporter, self).setUp() @@ -31,4 +32,7 @@ def test_create_database(self): r1.report_now() if patch.call_count != 2: raise AssertionError( - "Expected 2 calls to 'urlopen'. Received: {}".format(patch.call_count)) + "Expected 2 calls to 'urlopen'. Received: {}".format( + patch.call_count + ) + ) diff --git a/tests/test__meter.py b/tests/test__meter.py index 1d4b331..6b5cdb1 100644 --- a/tests/test__meter.py +++ b/tests/test__meter.py @@ -3,7 +3,6 @@ class MeterTestCase(TimedTestCase): - def setUp(self): super(MeterTestCase, self).setUp() self.meter = Meter(TimedTestCase.clock) @@ -17,18 +16,19 @@ def test__one_minute_rate(self): self.meter.tick() # the EWMA has a rate of 0.6 events/sec after the first tick - self.assertAlmostEqual( - 0.6, self.meter.get_one_minute_rate(), delta=0.000001) + self.assertAlmostEqual(0.6, self.meter.get_one_minute_rate(), delta=0.000001) self.clock.add(60) # the EWMA has a rate of 0.22072766 events/sec after 1 minute self.assertAlmostEqual( - 0.22072766, self.meter.get_one_minute_rate(), delta=0.000001) + 0.22072766, self.meter.get_one_minute_rate(), delta=0.000001 + ) self.clock.add(60) # the EWMA has a rate of 0.08120117 events/sec after 2 minute self.assertAlmostEqual( - 0.08120117, self.meter.get_one_minute_rate(), delta=0.000001) + 0.08120117, self.meter.get_one_minute_rate(), delta=0.000001 + ) def test__five_minute_rate(self): self.meter.mark(3) @@ -36,18 +36,19 @@ def test__five_minute_rate(self): self.meter.tick() # the EWMA has a rate of 0.6 events/sec after the first tick - self.assertAlmostEqual( - 0.6, self.meter.get_five_minute_rate(), delta=0.000001) + self.assertAlmostEqual(0.6, self.meter.get_five_minute_rate(), delta=0.000001) self.clock.add(60) # the EWMA has a rate of 0.49123845 events/sec after 1 minute self.assertAlmostEqual( - 0.49123845, self.meter.get_five_minute_rate(), delta=0.000001) + 0.49123845, self.meter.get_five_minute_rate(), delta=0.000001 + ) self.clock.add(60) # the EWMA has a rate of 0.40219203 events/sec after 2 minute self.assertAlmostEqual( - 0.40219203, self.meter.get_five_minute_rate(), delta=0.000001) + 0.40219203, self.meter.get_five_minute_rate(), delta=0.000001 + ) def test__fifteen_minute_rate(self): self.meter.mark(3) @@ -56,17 +57,20 @@ def test__fifteen_minute_rate(self): # the EWMA has a rate of 0.6 events/sec after the first tick self.assertAlmostEqual( - 0.6, self.meter.get_fifteen_minute_rate(), delta=0.000001) + 0.6, self.meter.get_fifteen_minute_rate(), delta=0.000001 + ) self.clock.add(60) # the EWMA has a rate of 0.56130419 events/sec after 1 minute self.assertAlmostEqual( - 0.56130419, self.meter.get_fifteen_minute_rate(), delta=0.000001) + 0.56130419, self.meter.get_fifteen_minute_rate(), delta=0.000001 + ) self.clock.add(60) # the EWMA has a rate of 0.52510399 events/sec after 2 minute self.assertAlmostEqual( - 0.52510399, self.meter.get_fifteen_minute_rate(), delta=0.000001) + 0.52510399, self.meter.get_fifteen_minute_rate(), delta=0.000001 + ) def test__mean_rate(self): self.meter.mark(60) diff --git a/tests/test__moving_average.py b/tests/test__moving_average.py index f34b7cb..5a547e9 100644 --- a/tests/test__moving_average.py +++ b/tests/test__moving_average.py @@ -3,7 +3,6 @@ class EWMATests(TimedTestCase): - def test_one_minute_EWMA_five_sec_tick(self): self.ewma = ExpWeightedMovingAvg(1, clock=self.clock) @@ -11,9 +10,18 @@ def test_one_minute_EWMA_five_sec_tick(self): self.clock.add(5) self.ewma.tick() - for expected_rate in [0.6, 0.22072766, 0.08120117, 0.02987224, - 0.01098938, 0.00404277, 0.00148725, - 0.00054713, 0.00020128, 0.00007405]: + for expected_rate in [ + 0.6, + 0.22072766, + 0.08120117, + 0.02987224, + 0.01098938, + 0.00404277, + 0.00148725, + 0.00054713, + 0.00020128, + 0.00007405, + ]: self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) self.clock.add(60) @@ -24,9 +32,18 @@ def test_five_minute_EWMA_five_sec_tick(self): self.clock.add(5) self.ewma.tick() - for expected_rate in [0.6, 0.49123845, 0.40219203, 0.32928698, - 0.26959738, 0.22072766, 0.18071653, - 0.14795818, 0.12113791, 0.09917933]: + for expected_rate in [ + 0.6, + 0.49123845, + 0.40219203, + 0.32928698, + 0.26959738, + 0.22072766, + 0.18071653, + 0.14795818, + 0.12113791, + 0.09917933, + ]: self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) self.clock.add(60) @@ -37,9 +54,18 @@ def test_fifteen_minute_EWMA_five_sec_tick(self): self.clock.add(5) self.ewma.tick() - for expected_rate in [0.6, 0.56130419, 0.52510399, 0.49123845, - 0.45955700, 0.42991879, 0.40219203, - 0.37625345, 0.35198773, 0.32928698]: + for expected_rate in [ + 0.6, + 0.56130419, + 0.52510399, + 0.49123845, + 0.45955700, + 0.42991879, + 0.40219203, + 0.37625345, + 0.35198773, + 0.32928698, + ]: self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) self.clock.add(60) @@ -49,9 +75,18 @@ def test_one_minute_EWMA_one_minute_tick(self): self.clock.add(5) self.ewma.tick() - for expected_rate in [0.6, 0.22072766, 0.08120117, 0.02987224, - 0.01098938, 0.00404277, 0.00148725, - 0.00054713, 0.00020128, 0.00007405]: + for expected_rate in [ + 0.6, + 0.22072766, + 0.08120117, + 0.02987224, + 0.01098938, + 0.00404277, + 0.00148725, + 0.00054713, + 0.00020128, + 0.00007405, + ]: self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) self.clock.add(60) @@ -62,9 +97,18 @@ def test_five_minute_EWMA_one_minute_tick(self): self.clock.add(5) self.ewma.tick() - for expected_rate in [0.6, 0.49123845, 0.40219203, 0.32928698, - 0.26959738, 0.22072766, 0.18071653, - 0.14795818, 0.12113791, 0.09917933]: + for expected_rate in [ + 0.6, + 0.49123845, + 0.40219203, + 0.32928698, + 0.26959738, + 0.22072766, + 0.18071653, + 0.14795818, + 0.12113791, + 0.09917933, + ]: self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) self.clock.add(60) @@ -75,8 +119,17 @@ def test_fifteen_minute_EWMA_one_minute_tick(self): self.clock.add(5) self.ewma.tick() - for expected_rate in [0.6, 0.56130419, 0.52510399, 0.49123845, - 0.45955700, 0.42991879, 0.40219203, - 0.37625345, 0.35198773, 0.32928698]: + for expected_rate in [ + 0.6, + 0.56130419, + 0.52510399, + 0.49123845, + 0.45955700, + 0.42991879, + 0.40219203, + 0.37625345, + 0.35198773, + 0.32928698, + ]: self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) self.clock.add(60) diff --git a/tests/test__newrelic_reporter.py b/tests/test__newrelic_reporter.py index 2454646..ab92f9f 100644 --- a/tests/test__newrelic_reporter.py +++ b/tests/test__newrelic_reporter.py @@ -17,8 +17,12 @@ def tearDown(self): def test_report_now(self): r = NewRelicReporter( - 'license_key', - registry=self.registry, reporting_interval=1, clock=self.clock, name='foo') + "license_key", + registry=self.registry, + reporting_interval=1, + clock=self.clock, + name="foo", + ) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) @@ -33,7 +37,10 @@ def test_report_now(self): c2.dec() self.clock.add(1) output = r.collect_metrics(self.registry) - expected = '{"agent": {"host": "%s", "pid": %s, "version": "%s"}, "components": [{"duration": 1, "guid": "com.github.pyformance", "metrics": {"Component/t1": {' \ - '"count": 1, "max": 1, "min": 1, "sum_of_squares": 1, "total": 1}}, "name": "foo"}]}' % (socket.gethostname(), os.getpid(), __version__) + expected = ( + '{"agent": {"host": "%s", "pid": %s, "version": "%s"}, "components": [{"duration": 1, "guid": "com.github.pyformance", "metrics": {"Component/t1": {' + '"count": 1, "max": 1, "min": 1, "sum_of_squares": 1, "total": 1}}, "name": "foo"}]}' + % (socket.gethostname(), os.getpid(), __version__) + ) self.assertEqual(expected.replace(".0", ""), output.replace(".0", "")) diff --git a/tests/test__opentsdb_reporter.py b/tests/test__opentsdb_reporter.py index 8fee63a..7ebf959 100644 --- a/tests/test__opentsdb_reporter.py +++ b/tests/test__opentsdb_reporter.py @@ -17,8 +17,15 @@ def tearDown(self): super(TestOpenTSDBReporter, self).tearDown() def test_report_now(self): - r = OpenTSDBReporter(application_name="app", write_key="key", registry=self.registry, reporting_interval=1, - clock=self.clock, prefix="prefix.", url="http://opentsdb.com/api/put") + r = OpenTSDBReporter( + application_name="app", + write_key="key", + registry=self.registry, + reporting_interval=1, + clock=self.clock, + prefix="prefix.", + url="http://opentsdb.com/api/put", + ) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) @@ -35,11 +42,18 @@ def test_report_now(self): output = r._collect_metrics(registry=self.registry) self.assertEqual(len(output), 31) for data in output: - assert data['metric'].startswith("prefix.") + assert data["metric"].startswith("prefix.") def test_send_request(self): - r = OpenTSDBReporter(application_name="app", write_key="key", registry=self.registry, reporting_interval=1, - clock=self.clock, prefix="prefix.", url="http://opentsdb.com/api/put") + r = OpenTSDBReporter( + application_name="app", + write_key="key", + registry=self.registry, + reporting_interval=1, + clock=self.clock, + prefix="prefix.", + url="http://opentsdb.com/api/put", + ) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) @@ -53,6 +67,8 @@ def test_send_request(self): c2.dec() c2.dec() self.clock.add(1) - with mock.patch("pyformance.reporters.opentsdb_reporter.urllib.urlopen") as patch: + with mock.patch( + "pyformance.reporters.opentsdb_reporter.urllib.urlopen" + ) as patch: r.report_now() patch.assert_called() diff --git a/tests/test__registry.py b/tests/test__registry.py index 70756fd..43a4c54 100644 --- a/tests/test__registry.py +++ b/tests/test__registry.py @@ -4,8 +4,6 @@ class RegistryTestCase(TimedTestCase): - - def setUp(self): super(RegistryTestCase, self).setUp() self.registry = MetricsRegistry(TimedTestCase.clock) @@ -14,4 +12,4 @@ def tearDown(self): super(RegistryTestCase, self).tearDown() def test__add(self): - self.registry.add('foo', Meter(TimedTestCase.clock)) + self.registry.add("foo", Meter(TimedTestCase.clock)) diff --git a/tests/test__syslog_reporter.py b/tests/test__syslog_reporter.py index 1597eba..8beda03 100644 --- a/tests/test__syslog_reporter.py +++ b/tests/test__syslog_reporter.py @@ -7,7 +7,6 @@ class TestSysLogReporter(TimedTestCase): - def setUp(self): super(TestSysLogReporter, self).setUp() self.registry = MetricsRegistry(clock=self.clock) @@ -20,7 +19,9 @@ def tearDown(self): def test_report_now(self): # connect to a local rsyslog server - r = SysLogReporter(registry=self.registry, reporting_interval=1, clock=self.clock) + r = SysLogReporter( + registry=self.registry, reporting_interval=1, clock=self.clock + ) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) @@ -36,14 +37,16 @@ def test_report_now(self): c2.dec() c2.dec() self.clock.add(1) - with mock.patch("pyformance.reporters.syslog_reporter.logging.Logger.info") as patch: + with mock.patch( + "pyformance.reporters.syslog_reporter.logging.Logger.info" + ) as patch: r.report_now() expected = '{"c1.count": 1, "counter-2.count": -2, "gcb.value": 123, "gsimple.value": 42, "hist.75_percentile": 160.0, "hist.95_percentile": 512, "hist.999_percentile": 512, "hist.99_percentile": 512, "hist.avg": 102.3, "hist.count": 10.0, "hist.max": 512, "hist.min": 1, "hist.std_dev": 164.94851048466944, "m1.15m_rate": 0, "m1.1m_rate": 0, "m1.5m_rate": 0, "m1.count": 1.0, "m1.mean_rate": 1.0, "t1.15m_rate": 0, "t1.1m_rate": 0, "t1.50_percentile": 1, "t1.5m_rate": 0, "t1.75_percentile": 1, "t1.95_percentile": 1, "t1.999_percentile": 1, "t1.99_percentile": 1, "t1.avg": 1.0, "t1.count": 1.0, "t1.max": 1, "t1.mean_rate": 1.0, "t1.min": 1, "t1.std_dev": 0.0, "t1.sum": 1.0, "timestamp": 1}' - + patch.assert_called_with(expected) if __name__ == "__main__": import unittest - unittest.main() + unittest.main() diff --git a/tests/test__timer.py b/tests/test__timer.py index 4aec85f..147604f 100644 --- a/tests/test__timer.py +++ b/tests/test__timer.py @@ -3,7 +3,6 @@ class TimerTestCase(TimedTestCase): - def setUp(self): super(TimerTestCase, self).setUp() self.timer = Timer()