From 6f004037665dad7bd503d13661b04a8fa9563dae Mon Sep 17 00:00:00 2001 From: Anton Bannikov Date: Thu, 19 Apr 2018 17:01:05 +0500 Subject: [PATCH 01/28] Added some spaces to correctly show in html I found that YAML format examples in documentation at this page was written incorrectly because of spaces, so users had errors when run it. I fixed errors and viewed Preview changes. So I hope now it looks better. --- docs/tutorial.rst | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index c4dcfc416..0fc2f7c84 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -14,16 +14,16 @@ Create a file on a server with Yandex.Tank: **load.yaml** .. code-block:: yaml phantom: - address: 203.0.113.1:80 # [Target's address]:[target's port] - uris: - - / - load_profile: - load_type: rps # schedule load by defining requests per second - schedule: line(1, 10, 10m) # starting from 1rps growing linearly to 10rps during 10 minutes - console: - enabled: true # enable console output - telegraf: - enabled: false # let's disable telegraf monitoring for the first time + address: 203.0.113.1:80 # [Target's address]:[target's port] + uris: + - / + load_profile: + load_type: rps # schedule load by defining requests per second + schedule: line(1, 10, 10m) # starting from 1rps growing linearly to 10rps during 10 minutes + console: + enabled: true # enable console output + telegraf: + enabled: false # let's disable telegraf monitoring for the first time And run: ``$ yandex-tank -c load.yaml`` @@ -74,17 +74,17 @@ have following lines: .. code-block:: yaml phantom: - address: 203.0.113.1:80 # [Target's address]:[target's port] - uris: - - /uri1 - - /uri2 - load_profile: - load_type: rps # schedule load by defining requests per second - schedule: const(10, 10m) # starting from 1rps growing linearly to 10rps during 10 minutes - console: - enabled: true # enable console output - telegraf: - enabled: false # let's disable telegraf monitoring for the first time + address: 203.0.113.1:80 # [Target's address]:[target's port] + uris: + - /uri1 + - /uri2 + load_profile: + load_type: rps # schedule load by defining requests per second + schedule: const(10, 10m) # starting from 1rps growing linearly to 10rps during 10 minutes + console: + enabled: true # enable console output + telegraf: + enabled: false # let's disable telegraf monitoring for the first time Preparing requests @@ -599,7 +599,7 @@ that. load.yaml: .. code-block:: yaml - phantom: + phantom: address: 203.0.113.1:80 load_profile: load_type: rps From 54b04f206908158b2b8963d41a801f4598917907 Mon Sep 17 00:00:00 2001 From: ival83 Date: Fri, 20 Sep 2019 18:12:07 +0300 Subject: [PATCH 02/28] support of http source for test resources --- yandextank/plugins/Pandora/config/schema.yaml | 9 +++++- yandextank/plugins/Pandora/plugin.py | 32 ++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/yandextank/plugins/Pandora/config/schema.yaml b/yandextank/plugins/Pandora/config/schema.yaml index 3a32d0e2c..fc1b1a5a4 100644 --- a/yandextank/plugins/Pandora/config/schema.yaml +++ b/yandextank/plugins/Pandora/config/schema.yaml @@ -6,7 +6,7 @@ affinity: pandora_cmd: type: string default: pandora - description: Pandora executable path + description: Pandora executable path or link to it buffered_seconds: type: integer default: 2 @@ -21,6 +21,13 @@ config_file: expvar: type: boolean default: false +resources: + default: [] + type: list + description: additional resources you need to download befor test +resource: + type: dict + description: dict with attributes for additional resources report_file: type: string nullable: true diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 8abfbe75e..a93083313 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -2,6 +2,7 @@ import logging import subprocess import time +import os from threading import Event import yaml @@ -43,6 +44,7 @@ def __init__(self, core, cfg, name): self.__schedule = None self.ammofile = None self.process_stderr_file = None + self.resources = [] @staticmethod def get_key(): @@ -56,11 +58,15 @@ def get_available_options(self): return opts def configure(self): - self.pandora_cmd = self.get_option("pandora_cmd") + #self.pandora_cmd = self.get_option("pandora_cmd") self.report_file = self.get_option("report_file") self.buffered_seconds = self.get_option("buffered_seconds") self.affinity = self.get_option("affinity", "") + self.resources = self.get_option("resources") + #if we use custom pandora binary, we can download it and make it executable + self.pandora_cmd = self.get_resource(self.get_option("pandora_cmd"), "./pandora", permissions=0755) + # get config_contents and patch it: expand resources via resource manager # config_content option has more priority over config_file if self.get_option("config_content"): @@ -75,6 +81,11 @@ def configure(self): raise RuntimeError("Neither pandora.config_content, nor pandora.config_file specified") logger.debug('Config after parsing for patching: %s', self.config_contents) + #download all resources from self.get_options("resources") + if len(self.resources) > 0: + for resource in self.resources: + self.get_resource(resource["src"],resource["dst"]) + # find report filename and add to artifacts self.report_file = self.__find_report_filename() with open(self.report_file, 'w'): @@ -186,6 +197,24 @@ def get_stats_reader(self): self.stats_reader = PandoraStatsReader(self.expvar_enabled, self.expvar_port) return self.stats_reader + def get_resource(self, resource, dst, permissions=0644): + opener = resource_manager.get_opener(resource) + if isinstance(opener, HttpOpener): + tmp_path = opener.download_file(True, try_ungzip=True) + try: + os.rename(tmp_path, dst) + logger.info('Successfully moved resource %s', dst) + except OSError as ex: + logger.debug("Could not move resource %s\n%s", dst, ex) + else: + dst = opener.get_filename + try: + os.chmod(dst, permissions) + logger.info('Permissions on %s have changed %d', dst, permissions) + except OSError as ex: + logger.debug("Could not chane pepermissions on %s\n%s", dst, ex) + return dst + def prepare_test(self): try: console = self.core.get_plugin_of_type(ConsolePlugin) @@ -199,6 +228,7 @@ def prepare_test(self): self.core.job.aggregator.add_result_listener(widget) def start_test(self): + print self.pandora_cmd args = [self.pandora_cmd] +\ (['-expvar'] if self.expvar else []) +\ [self.pandora_config_file] From 291cc614833af1def785df281f6a9278bb2ade8d Mon Sep 17 00:00:00 2001 From: ival83 Date: Fri, 20 Sep 2019 18:16:22 +0300 Subject: [PATCH 03/28] remove comment --- yandextank/plugins/Pandora/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index a93083313..511ecc66a 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -58,7 +58,6 @@ def get_available_options(self): return opts def configure(self): - #self.pandora_cmd = self.get_option("pandora_cmd") self.report_file = self.get_option("report_file") self.buffered_seconds = self.get_option("buffered_seconds") self.affinity = self.get_option("affinity", "") From 1b381e03ac2d3dfd969b344b6b8f9327417fcf30 Mon Sep 17 00:00:00 2001 From: ival83 Date: Fri, 20 Sep 2019 19:45:03 +0300 Subject: [PATCH 04/28] code style --- yandextank/plugins/Pandora/plugin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 511ecc66a..2edcb60ee 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -63,9 +63,9 @@ def configure(self): self.affinity = self.get_option("affinity", "") self.resources = self.get_option("resources") - #if we use custom pandora binary, we can download it and make it executable + # if we use custom pandora binary, we can download it and make it executable self.pandora_cmd = self.get_resource(self.get_option("pandora_cmd"), "./pandora", permissions=0755) - + # get config_contents and patch it: expand resources via resource manager # config_content option has more priority over config_file if self.get_option("config_content"): @@ -80,10 +80,10 @@ def configure(self): raise RuntimeError("Neither pandora.config_content, nor pandora.config_file specified") logger.debug('Config after parsing for patching: %s', self.config_contents) - #download all resources from self.get_options("resources") + # download all resources from self.get_options("resources") if len(self.resources) > 0: for resource in self.resources: - self.get_resource(resource["src"],resource["dst"]) + self.get_resource(resource["src"], resource["dst"]) # find report filename and add to artifacts self.report_file = self.__find_report_filename() From 6cf3749cd68f94cdc637943f36002def43de0935 Mon Sep 17 00:00:00 2001 From: ival83 Date: Fri, 20 Sep 2019 19:50:56 +0300 Subject: [PATCH 05/28] debug prints removed --- yandextank/plugins/Pandora/config/schema.yaml | 2 +- yandextank/plugins/Pandora/plugin.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/yandextank/plugins/Pandora/config/schema.yaml b/yandextank/plugins/Pandora/config/schema.yaml index fc1b1a5a4..d9a0012f9 100644 --- a/yandextank/plugins/Pandora/config/schema.yaml +++ b/yandextank/plugins/Pandora/config/schema.yaml @@ -24,7 +24,7 @@ expvar: resources: default: [] type: list - description: additional resources you need to download befor test + description: additional resources you need to download before test resource: type: dict description: dict with attributes for additional resources diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 2edcb60ee..cdf57e15c 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -227,7 +227,6 @@ def prepare_test(self): self.core.job.aggregator.add_result_listener(widget) def start_test(self): - print self.pandora_cmd args = [self.pandora_cmd] +\ (['-expvar'] if self.expvar else []) +\ [self.pandora_config_file] From b763da6845cd86c8d4440b34609d0b2f0b3ede42 Mon Sep 17 00:00:00 2001 From: Mikhail Dyomin Date: Wed, 25 Sep 2019 18:29:29 +0300 Subject: [PATCH 06/28] Fix _stop_agent() for localhost --- yandextank/plugins/Telegraf/client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/yandextank/plugins/Telegraf/client.py b/yandextank/plugins/Telegraf/client.py index b4ca92502..aa5a0b310 100644 --- a/yandextank/plugins/Telegraf/client.py +++ b/yandextank/plugins/Telegraf/client.py @@ -138,13 +138,16 @@ def read_buffer(self): 'otherwise something really nasty happend', exc_info=True) + def _stop_agent(self): + if self.session: + self.session.terminate() + logger.info('Waiting localhost agent to terminate...') + def uninstall(self): """ Remove agent's files from remote host """ if self.session: - logger.info('Waiting monitoring data...') - self.session.terminate() self.session.wait() self.session = None log_filename = "agent_{host}.log".format(host="localhost") From 5c4d48cdda75c80ad4c6feadf81fcf3a4d52004f Mon Sep 17 00:00:00 2001 From: Nurlan Date: Wed, 25 Sep 2019 20:46:04 +0300 Subject: [PATCH 07/28] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e02528958..3b1785239 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='yandextank', - version='1.12.5', + version='1.12.5.1', description='a performance measurement tool', longer_description=''' Yandex.Tank is a performance measurement and load testing automatization tool. From 0a99feab92f9e00f59eda54256a97919865b529a Mon Sep 17 00:00:00 2001 From: Alexey Lavrenuke Date: Tue, 8 Oct 2019 18:57:04 +0300 Subject: [PATCH 08/28] Update setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3b1785239..4de33b8f8 100644 --- a/setup.py +++ b/setup.py @@ -10,8 +10,8 @@ load generation and provides a common configuration system for them and analytic tools for the results they produce. ''', - maintainer='Alexey Lavrenuke (load testing)', - maintainer_email='direvius@yandex-team.ru', + maintainer='Nurlan Nugumanov', + maintainer_email='r2d2@yandex-team.ru', url='http://yandex.github.io/yandex-tank/', namespace_packages=["yandextank", "yandextank.plugins"], packages=find_packages(exclude=["tests", "tmp", "docs", "data"]), From 43edae5d561172bce73b1d8ab8eba48839992e55 Mon Sep 17 00:00:00 2001 From: szypulka Date: Wed, 9 Oct 2019 16:19:55 +0300 Subject: [PATCH 09/28] Change maintainer to load team --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4de33b8f8..6d8d21bea 100644 --- a/setup.py +++ b/setup.py @@ -10,8 +10,8 @@ load generation and provides a common configuration system for them and analytic tools for the results they produce. ''', - maintainer='Nurlan Nugumanov', - maintainer_email='r2d2@yandex-team.ru', + maintainer='Yandex Load Team', + maintainer_email='load@yandex-team.ru', url='http://yandex.github.io/yandex-tank/', namespace_packages=["yandextank", "yandextank.plugins"], packages=find_packages(exclude=["tests", "tmp", "docs", "data"]), From 2b7fb9aca5fce6da55be4fbafed7a18247a410da Mon Sep 17 00:00:00 2001 From: tmaitz Date: Fri, 11 Oct 2019 13:29:24 +0300 Subject: [PATCH 10/28] cast int metrics to float. avoid influx db cast exception --- yandextank/plugins/InfluxUploader/decoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yandextank/plugins/InfluxUploader/decoder.py b/yandextank/plugins/InfluxUploader/decoder.py index bd9059e92..c69a10a3c 100644 --- a/yandextank/plugins/InfluxUploader/decoder.py +++ b/yandextank/plugins/InfluxUploader/decoder.py @@ -49,7 +49,8 @@ def decode_monitoring(self, data): {"host": host, "comment": host_data.get("comment")}, second_data["timestamp"], { - metric: value + # cast int to float. avoid https://github.com/yandex/yandex-tank/issues/776 + metric: float(value) if type(value) is int else value for metric, value in host_data["metrics"].iteritems() } ) From c43f96d1a583f801e6a90c98f01a9e421a52f8f2 Mon Sep 17 00:00:00 2001 From: szypulka Date: Fri, 11 Oct 2019 14:42:57 +0300 Subject: [PATCH 11/28] Add autostop rps and reason to status --- yandextank/plugins/Autostop/plugin.py | 12 ++++++++++-- yandextank/plugins/DataUploader/plugin.py | 9 +-------- yandextank/plugins/NeUploader/plugin.py | 5 +++++ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/yandextank/plugins/Autostop/plugin.py b/yandextank/plugins/Autostop/plugin.py index b267eb8cc..dd6507a3f 100644 --- a/yandextank/plugins/Autostop/plugin.py +++ b/yandextank/plugins/Autostop/plugin.py @@ -20,6 +20,7 @@ def __init__(self, core, cfg, name): AggregateResultListener.__init__(self) self.cause_criterion = None + self.imbalance_rps = 0 self._criterions = {} self.custom_criterions = [] self.counting = [] @@ -112,9 +113,16 @@ def on_aggregated_data(self, data, stat): if not self.cause_criterion: for criterion_text, criterion in self._criterions.iteritems(): if criterion.notify(data, stat): - self.log.warning( - "Autostop criterion requested test stop: %s", criterion_text) self.cause_criterion = criterion + if self.cause_criterion.cause_second: + self.imbalance_rps = int(self.cause_criterion.cause_second[1]["metrics"]["reqps"]) + if not self.imbalance_rps: + self.imbalance_rps = int( + self.cause_criterion.cause_second[0]["overall"]["interval_real"]["len"]) + self.core.publish('autostop', 'rps', self.imbalance_rps) + self.core.publish('autostop', 'reason', criterion.explain()) + self.log.warning( + "Autostop criterion requested test stop on %d rps: %s", self.imbalance_rps, criterion_text) open(self._stop_report_path, 'w').write(criterion_text) self.core.add_artifact_file(self._stop_report_path) diff --git a/yandextank/plugins/DataUploader/plugin.py b/yandextank/plugins/DataUploader/plugin.py index e672052f9..b9f6f70d5 100644 --- a/yandextank/plugins/DataUploader/plugin.py +++ b/yandextank/plugins/DataUploader/plugin.py @@ -361,15 +361,8 @@ def post_process(self, rc): logger.debug("No autostop plugin loaded", exc_info=True) if autostop and autostop.cause_criterion: - rps = 0 - if autostop.cause_criterion.cause_second: - rps = autostop.cause_criterion.cause_second[ - 1]["metrics"]["reqps"] - if not rps: - rps = autostop.cause_criterion.cause_second[0][ - "overall"]["interval_real"]["len"] self.lp_job.set_imbalance_and_dsc( - int(rps), autostop.cause_criterion.explain()) + autostop.imbalance_rps, autostop.cause_criterion.explain()) else: logger.debug("No autostop cause detected") diff --git a/yandextank/plugins/NeUploader/plugin.py b/yandextank/plugins/NeUploader/plugin.py index 3f566ef11..680686894 100644 --- a/yandextank/plugins/NeUploader/plugin.py +++ b/yandextank/plugins/NeUploader/plugin.py @@ -68,6 +68,11 @@ def data_session(self): def _cleanup(self): uploader_metainfo = self.map_uploader_tags(self.core.status.get('uploader')) self.data_session.update_job(uploader_metainfo) + if self.core.status.get('autostop'): + autostop_rps = self.core.status['autostop'].get('rps', 0) + autostop_reason = self.core.status['autostop'].get('reason', '') + self.log.warning('Autostop: %s %s', autostop_rps, autostop_reason) + self.data_session.update_job({'autostop_rps': autostop_rps, 'autostop_reason': autostop_reason}) self.data_session.close(test_end=self.core.status.get('generator', {}).get('test_end', 0) * 10**6) def is_test_finished(self): From ee7a698da1941faa5bdfce65e4a93b53ca36dfef Mon Sep 17 00:00:00 2001 From: Joulan Date: Sat, 12 Oct 2019 14:02:33 +0300 Subject: [PATCH 12/28] pr comment fix and test for metrics cast --- yandextank/plugins/InfluxUploader/decoder.py | 2 +- .../InfluxUploader/tests/test_decoder.py | 65 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 yandextank/plugins/InfluxUploader/tests/test_decoder.py diff --git a/yandextank/plugins/InfluxUploader/decoder.py b/yandextank/plugins/InfluxUploader/decoder.py index c69a10a3c..c38530843 100644 --- a/yandextank/plugins/InfluxUploader/decoder.py +++ b/yandextank/plugins/InfluxUploader/decoder.py @@ -50,7 +50,7 @@ def decode_monitoring(self, data): second_data["timestamp"], { # cast int to float. avoid https://github.com/yandex/yandex-tank/issues/776 - metric: float(value) if type(value) is int else value + metric: float(value) if isinstance(value, int) else value for metric, value in host_data["metrics"].iteritems() } ) diff --git a/yandextank/plugins/InfluxUploader/tests/test_decoder.py b/yandextank/plugins/InfluxUploader/tests/test_decoder.py new file mode 100644 index 000000000..0ac12d06a --- /dev/null +++ b/yandextank/plugins/InfluxUploader/tests/test_decoder.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +from uuid import uuid4 +from yandextank.plugins.InfluxUploader.decoder import Decoder + + +class TestDecoder(object): + def test_metrics_cast(self): + test_uuid = str(uuid4()) + tank_tag = 'test_tank_tag' + comment = 'test comment' + raw_metrics = { + 'metric1': -123, + 'metric2': -123.456, + 'metric3': 123, + 'metric4': 123.456, + 'metric5': 0, + 'metric6': -0.1, + 'metric7': 0.1, + 'metric8': 'down', + } + timestamp = 123456789 + host = '127.0.0.1' + data = [ + { + 'data': { + host: { + 'comment': comment, + 'metrics': raw_metrics + } + }, + 'timestamp': timestamp + } + ] + expected_metrics = { + 'metric1': -123.0, + 'metric2': -123.456, + 'metric3': 123.0, + 'metric4': 123.456, + 'metric5': 0.0, + 'metric6': -0.1, + 'metric7': 0.1, + 'metric8': 'down' + } + + decoder = Decoder(tank_tag, test_uuid, {}, True, True) + result_points = decoder.decode_monitoring(data) + + assert (len(result_points) == 1) + r_point = result_points[0] + # check other props + assert (r_point['time'] == timestamp) + assert (r_point['measurement'] == 'monitoring') + assert (r_point['tags']['comment'] == comment) + assert (r_point['tags']['host'] == host) + assert (r_point['tags']['tank'] == tank_tag) + assert (r_point['tags']['uuid'] == test_uuid) + # check metric cast + assert (len(r_point['fields']) == len(expected_metrics)) + for metric, value in r_point['fields'].iteritems(): + if metric not in expected_metrics: + assert False + if not isinstance(value, type(expected_metrics[metric])): + assert False + if not value == expected_metrics[metric]: + assert False From ecf8fda42adafab9b6025961a922a1e781624b2e Mon Sep 17 00:00:00 2001 From: fomars Date: Tue, 15 Oct 2019 16:23:37 +0300 Subject: [PATCH 13/28] fix luna test name --- yandextank/plugins/NeUploader/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yandextank/plugins/NeUploader/plugin.py b/yandextank/plugins/NeUploader/plugin.py index e265f311e..00b20c38f 100644 --- a/yandextank/plugins/NeUploader/plugin.py +++ b/yandextank/plugins/NeUploader/plugin.py @@ -184,6 +184,6 @@ def map_uploader_tags(self, uploader_tags): return {} else: meta_tags_names = ['component', 'description', 'name', 'person', 'task', 'version', 'lunapark_jobno'] - meta_tags = {key: uploader_tags.get(key, self.cfg.get(key, '')) for key in meta_tags_names} + meta_tags = {key: uploader_tags.get(key) for key in meta_tags_names if key in uploader_tags} meta_tags.update({k: v if v is not None else '' for k, v in uploader_tags.get('meta', {}).items()}) return meta_tags From 3c7cc69fdcc646d2ce9545b27ff01495a575e80b Mon Sep 17 00:00:00 2001 From: fomars Date: Tue, 15 Oct 2019 16:23:37 +0300 Subject: [PATCH 14/28] fix luna test name --- yandextank/plugins/NeUploader/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yandextank/plugins/NeUploader/plugin.py b/yandextank/plugins/NeUploader/plugin.py index f6ac98483..6a131e35b 100644 --- a/yandextank/plugins/NeUploader/plugin.py +++ b/yandextank/plugins/NeUploader/plugin.py @@ -189,6 +189,6 @@ def map_uploader_tags(self, uploader_tags): return {} else: meta_tags_names = ['component', 'description', 'name', 'person', 'task', 'version', 'lunapark_jobno'] - meta_tags = {key: uploader_tags.get(key, self.cfg.get(key, '')) for key in meta_tags_names} + meta_tags = {key: uploader_tags.get(key) for key in meta_tags_names if key in uploader_tags} meta_tags.update({k: v if v is not None else '' for k, v in uploader_tags.get('meta', {}).items()}) return meta_tags From 0e5db0b8c715f6003fbe3d1219c3721c604270ff Mon Sep 17 00:00:00 2001 From: fomars Date: Tue, 15 Oct 2019 20:35:41 +0300 Subject: [PATCH 15/28] fix neuploader autostop --- yandextank/plugins/NeUploader/plugin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yandextank/plugins/NeUploader/plugin.py b/yandextank/plugins/NeUploader/plugin.py index 6a131e35b..a73a7405b 100644 --- a/yandextank/plugins/NeUploader/plugin.py +++ b/yandextank/plugins/NeUploader/plugin.py @@ -70,12 +70,12 @@ def data_session(self): def _cleanup(self): uploader_metainfo = self.map_uploader_tags(self.core.status.get('uploader')) - self.data_session.update_job(uploader_metainfo) if self.core.status.get('autostop'): - autostop_rps = self.core.status['autostop'].get('rps', 0) - autostop_reason = self.core.status['autostop'].get('reason', '') + autostop_rps = self.core.status.get('autostop', {}).get('rps', 0) + autostop_reason = self.core.status.get('autostop', {}).get('reason', '') self.log.warning('Autostop: %s %s', autostop_rps, autostop_reason) - self.data_session.update_job({'autostop_rps': autostop_rps, 'autostop_reason': autostop_reason}) + uploader_metainfo.update({'autostop_rps': autostop_rps, 'autostop_reason': autostop_reason}) + self.data_session.update_job(uploader_metainfo) self.data_session.close(test_end=self.core.status.get('generator', {}).get('test_end', 0) * 10**6) def is_test_finished(self): From afbfde62c1b468851397db4a9ee51adbdfd7ac39 Mon Sep 17 00:00:00 2001 From: ival83 Date: Mon, 21 Oct 2019 17:25:11 +0300 Subject: [PATCH 16/28] pandora config can be delivered through resources --- yandextank/plugins/Pandora/plugin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index cdf57e15c..3afb0bded 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -66,6 +66,11 @@ def configure(self): # if we use custom pandora binary, we can download it and make it executable self.pandora_cmd = self.get_resource(self.get_option("pandora_cmd"), "./pandora", permissions=0755) + # download all resources from self.get_options("resources") + if len(self.resources) > 0: + for resource in self.resources: + self.get_resource(resource["src"], resource["dst"]) + # get config_contents and patch it: expand resources via resource manager # config_content option has more priority over config_file if self.get_option("config_content"): @@ -80,11 +85,6 @@ def configure(self): raise RuntimeError("Neither pandora.config_content, nor pandora.config_file specified") logger.debug('Config after parsing for patching: %s', self.config_contents) - # download all resources from self.get_options("resources") - if len(self.resources) > 0: - for resource in self.resources: - self.get_resource(resource["src"], resource["dst"]) - # find report filename and add to artifacts self.report_file = self.__find_report_filename() with open(self.report_file, 'w'): From 8c3e3aff62f79253e74b227c54b294d13cfe3ca6 Mon Sep 17 00:00:00 2001 From: ival83 Date: Mon, 21 Oct 2019 17:34:54 +0300 Subject: [PATCH 17/28] pandora copies resources to avoid move problems --- yandextank/plugins/Pandora/plugin.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 3afb0bded..43a3f2bf1 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -3,6 +3,7 @@ import subprocess import time import os +import shutil from threading import Event import yaml @@ -201,9 +202,9 @@ def get_resource(self, resource, dst, permissions=0644): if isinstance(opener, HttpOpener): tmp_path = opener.download_file(True, try_ungzip=True) try: - os.rename(tmp_path, dst) + shutil.copy(tmp_path, dst) logger.info('Successfully moved resource %s', dst) - except OSError as ex: + except Exception as ex: logger.debug("Could not move resource %s\n%s", dst, ex) else: dst = opener.get_filename @@ -211,7 +212,7 @@ def get_resource(self, resource, dst, permissions=0644): os.chmod(dst, permissions) logger.info('Permissions on %s have changed %d', dst, permissions) except OSError as ex: - logger.debug("Could not chane pepermissions on %s\n%s", dst, ex) + logger.debug("Could not change pepermissions on %s\n%s", dst, ex) return dst def prepare_test(self): From d220b3637d7ad8a32fa88d6f11ce3553a4976400 Mon Sep 17 00:00:00 2001 From: ival83 Date: Tue, 22 Oct 2019 14:43:11 +0300 Subject: [PATCH 18/28] stop tank if it can't prepare resources --- yandextank/plugins/Pandora/plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 43a3f2bf1..f146f1da3 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -206,6 +206,7 @@ def get_resource(self, resource, dst, permissions=0644): logger.info('Successfully moved resource %s', dst) except Exception as ex: logger.debug("Could not move resource %s\n%s", dst, ex) + raise RuntimeError('Unable to prepare resource') else: dst = opener.get_filename try: @@ -213,6 +214,7 @@ def get_resource(self, resource, dst, permissions=0644): logger.info('Permissions on %s have changed %d', dst, permissions) except OSError as ex: logger.debug("Could not change pepermissions on %s\n%s", dst, ex) + raise RuntimeError('Unable to prepare resource') return dst def prepare_test(self): From 3791396d55c25d71de0fc2d769de9532d516bb31 Mon Sep 17 00:00:00 2001 From: fomars Date: Mon, 28 Oct 2019 15:32:47 +0300 Subject: [PATCH 19/28] tag all metrics in test --- setup.py | 2 +- yandextank/plugins/NeUploader/config/schema.yaml | 6 +++++- yandextank/plugins/NeUploader/plugin.py | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index be817ed0c..4fd3fbb51 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ 'psutil>=1.2.1', 'requests>=2.5.1', 'paramiko>=1.16.0', 'pandas==0.24.2', 'numpy==1.15.4', 'future>=0.16.0', 'pip>=8.1.2', - 'pyyaml>=4.2b1', 'cerberus==1.2', 'influxdb>=5.0.0', 'netort>=0.6.10', + 'pyyaml>=4.2b1', 'cerberus==1.3.1', 'influxdb>=5.0.0', 'netort>=0.6.10', 'retrying>=1.3.3', 'pytest-runner' ], setup_requires=[ diff --git a/yandextank/plugins/NeUploader/config/schema.yaml b/yandextank/plugins/NeUploader/config/schema.yaml index c22ea9e45..564256087 100644 --- a/yandextank/plugins/NeUploader/config/schema.yaml +++ b/yandextank/plugins/NeUploader/config/schema.yaml @@ -9,4 +9,8 @@ db_name: test_name: description: test name type: string - default: my test \ No newline at end of file + default: my test +meta: + type: dict + keysrules: + forbidden: ['name', 'raw', 'aggregate', 'group', 'host', 'type'] \ No newline at end of file diff --git a/yandextank/plugins/NeUploader/plugin.py b/yandextank/plugins/NeUploader/plugin.py index a73a7405b..2a848bc9f 100644 --- a/yandextank/plugins/NeUploader/plugin.py +++ b/yandextank/plugins/NeUploader/plugin.py @@ -118,7 +118,8 @@ def get_metric_obj(self, col, case): raw=False, aggregate=True, source='tank', - importance='high' if col in self.importance_high else '' + importance='high' if col in self.importance_high else '', + **self.cfg.get('meta', {}) ) for col, constructor in self.col_map.items() } self.metrics_objs[case] = case_metrics @@ -153,7 +154,8 @@ def upload_monitoring(self, data): self.monitoring_metrics[metric_name] = self.data_session.new_true_metric(name, group=group, host=panel, - type='monitoring') + type='monitoring', + **self.cfg.get('meta', {})) self.monitoring_metrics[metric_name].put(df) @staticmethod From fd36dd52498a1d8269a3064e3ad430e4ddd2a3c6 Mon Sep 17 00:00:00 2001 From: ival83 Date: Tue, 5 Nov 2019 14:44:37 +0300 Subject: [PATCH 20/28] try/excepts removed from resource getter --- yandextank/plugins/Pandora/plugin.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index f146f1da3..b78dc417c 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -201,20 +201,12 @@ def get_resource(self, resource, dst, permissions=0644): opener = resource_manager.get_opener(resource) if isinstance(opener, HttpOpener): tmp_path = opener.download_file(True, try_ungzip=True) - try: - shutil.copy(tmp_path, dst) - logger.info('Successfully moved resource %s', dst) - except Exception as ex: - logger.debug("Could not move resource %s\n%s", dst, ex) - raise RuntimeError('Unable to prepare resource') + shutil.copy(tmp_path, dst) + logger.info('Successfully moved resource %s', dst) else: dst = opener.get_filename - try: - os.chmod(dst, permissions) - logger.info('Permissions on %s have changed %d', dst, permissions) - except OSError as ex: - logger.debug("Could not change pepermissions on %s\n%s", dst, ex) - raise RuntimeError('Unable to prepare resource') + os.chmod(dst, permissions) + logger.info('Permissions on %s have changed %d', dst, permissions) return dst def prepare_test(self): From 94d850ab7999b68eae96defe4512c3f894ad4c52 Mon Sep 17 00:00:00 2001 From: szypulka Date: Wed, 6 Nov 2019 13:58:46 +0300 Subject: [PATCH 21/28] Faster upload to luna (#811) * Bump netort version * Update requirements --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 4fd3fbb51..a2a059e29 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='yandextank', - version='1.12.5.1', + version='1.12.6', description='a performance measurement tool', longer_description=''' Yandex.Tank is a performance measurement and load testing automatization tool. @@ -20,8 +20,8 @@ 'psutil>=1.2.1', 'requests>=2.5.1', 'paramiko>=1.16.0', 'pandas==0.24.2', 'numpy==1.15.4', 'future>=0.16.0', 'pip>=8.1.2', - 'pyyaml>=4.2b1', 'cerberus==1.3.1', 'influxdb>=5.0.0', 'netort>=0.6.10', - 'retrying>=1.3.3', 'pytest-runner' + 'pyyaml>=4.2b1', 'cerberus==1.3.1', 'influxdb>=5.0.0', 'netort>=0.7.1', + 'retrying>=1.3.3', 'pytest-runner', 'typing' ], setup_requires=[ ], From 3a4dac483bdc7a639a5eef9e0ca632f19208754d Mon Sep 17 00:00:00 2001 From: fomars Date: Wed, 6 Nov 2019 22:13:01 +0300 Subject: [PATCH 22/28] add test to regression --- yandextank/plugins/NeUploader/plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yandextank/plugins/NeUploader/plugin.py b/yandextank/plugins/NeUploader/plugin.py index 2a848bc9f..d208068a2 100644 --- a/yandextank/plugins/NeUploader/plugin.py +++ b/yandextank/plugins/NeUploader/plugin.py @@ -64,8 +64,9 @@ def data_session(self): self._data_session = DataSession({'clients': self.clients_cfg}, test_start=self.core.status['generator']['test_start'] * 10**6) self.add_cleanup(self._cleanup) - self._data_session.update_job({'name': self.cfg.get('test_name'), - '__type': 'tank'}) + self._data_session.update_job(dict({'name': self.cfg.get('test_name'), + '__type': 'tank'}, + **self.cfg.get('meta', {}))) return self._data_session def _cleanup(self): From 9c2da5d90b056bdee9dbc6070340257ed8b87340 Mon Sep 17 00:00:00 2001 From: fomars Date: Fri, 15 Nov 2019 16:52:11 +0300 Subject: [PATCH 23/28] keep neuploader meta YANDEXTANK-230 --- yandextank/plugins/NeUploader/plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yandextank/plugins/NeUploader/plugin.py b/yandextank/plugins/NeUploader/plugin.py index d208068a2..6bb0c8b79 100644 --- a/yandextank/plugins/NeUploader/plugin.py +++ b/yandextank/plugins/NeUploader/plugin.py @@ -76,6 +76,7 @@ def _cleanup(self): autostop_reason = self.core.status.get('autostop', {}).get('reason', '') self.log.warning('Autostop: %s %s', autostop_rps, autostop_reason) uploader_metainfo.update({'autostop_rps': autostop_rps, 'autostop_reason': autostop_reason}) + uploader_metainfo.update(self.cfg.get('meta', {})) self.data_session.update_job(uploader_metainfo) self.data_session.close(test_end=self.core.status.get('generator', {}).get('test_end', 0) * 10**6) From e0fffc7c469cfd5dc7101c90b1df6458cef69a16 Mon Sep 17 00:00:00 2001 From: Ian Duffy Date: Thu, 21 Nov 2019 19:38:07 +0000 Subject: [PATCH 24/28] Add an OpenTSDB Uploader --- setup.py | 1 + yandextank/config_converter/converter.py | 1 + yandextank/core/config/00-base.yaml | 4 + ...st_decoder.py => test_influxdb_decoder.py} | 0 .../plugins/OpenTSDBUploader/__init__.py | 1 + .../OpenTSDBUploader/client/__init__.py | 1 + .../plugins/OpenTSDBUploader/client/client.py | 194 ++++++++++++++++++ .../OpenTSDBUploader/config/schema.yaml | 36 ++++ .../plugins/OpenTSDBUploader/decoder.py | 177 ++++++++++++++++ yandextank/plugins/OpenTSDBUploader/plugin.py | 90 ++++++++ .../tests/test_opentsdb_decoder.py | 63 ++++++ 11 files changed, 568 insertions(+) rename yandextank/plugins/InfluxUploader/tests/{test_decoder.py => test_influxdb_decoder.py} (100%) create mode 100644 yandextank/plugins/OpenTSDBUploader/__init__.py create mode 100644 yandextank/plugins/OpenTSDBUploader/client/__init__.py create mode 100644 yandextank/plugins/OpenTSDBUploader/client/client.py create mode 100644 yandextank/plugins/OpenTSDBUploader/config/schema.yaml create mode 100644 yandextank/plugins/OpenTSDBUploader/decoder.py create mode 100644 yandextank/plugins/OpenTSDBUploader/plugin.py create mode 100644 yandextank/plugins/OpenTSDBUploader/tests/test_opentsdb_decoder.py diff --git a/setup.py b/setup.py index a2a059e29..e9e78414f 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ 'yandextank.plugins.Console': ['config/*'], 'yandextank.plugins.DataUploader': ['config/*'], 'yandextank.plugins.InfluxUploader': ['config/*'], + 'yandextank.plugins.OpenTSDBUploader': ['config/*'], 'yandextank.plugins.JMeter': ['config/*'], 'yandextank.plugins.JsonReport': ['config/*'], 'yandextank.plugins.Pandora': ['config/*'], diff --git a/yandextank/config_converter/converter.py b/yandextank/config_converter/converter.py index 97e4c1b40..1556c182d 100644 --- a/yandextank/config_converter/converter.py +++ b/yandextank/config_converter/converter.py @@ -49,6 +49,7 @@ def parse_package_name(package_path): 'JsonReport': 'json_report|jsonreport', 'Pandora': 'pandora', 'InfluxUploader': 'influx', + 'OpenTSDBUploader': 'opentsdb', } diff --git a/yandextank/core/config/00-base.yaml b/yandextank/core/config/00-base.yaml index 0c8e98f76..8554604a4 100644 --- a/yandextank/core/config/00-base.yaml +++ b/yandextank/core/config/00-base.yaml @@ -40,3 +40,7 @@ overload: influx: enabled: false package: yandextank.plugins.InfluxUploader +opentsdb: + enabled: false + package: yandextank.plugins.OpenTSDBUploader + diff --git a/yandextank/plugins/InfluxUploader/tests/test_decoder.py b/yandextank/plugins/InfluxUploader/tests/test_influxdb_decoder.py similarity index 100% rename from yandextank/plugins/InfluxUploader/tests/test_decoder.py rename to yandextank/plugins/InfluxUploader/tests/test_influxdb_decoder.py diff --git a/yandextank/plugins/OpenTSDBUploader/__init__.py b/yandextank/plugins/OpenTSDBUploader/__init__.py new file mode 100644 index 000000000..eb4ac99ab --- /dev/null +++ b/yandextank/plugins/OpenTSDBUploader/__init__.py @@ -0,0 +1 @@ +from .plugin import Plugin # noqa:F401 diff --git a/yandextank/plugins/OpenTSDBUploader/client/__init__.py b/yandextank/plugins/OpenTSDBUploader/client/__init__.py new file mode 100644 index 000000000..e0eaac508 --- /dev/null +++ b/yandextank/plugins/OpenTSDBUploader/client/__init__.py @@ -0,0 +1 @@ +from .client import OpenTSDBClient # noqa:F401 diff --git a/yandextank/plugins/OpenTSDBUploader/client/client.py b/yandextank/plugins/OpenTSDBUploader/client/client.py new file mode 100644 index 000000000..2b94e2e6a --- /dev/null +++ b/yandextank/plugins/OpenTSDBUploader/client/client.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +"""Python client for OpenTSDB.""" + +import json +import random +import requests +import requests.exceptions +import time + + +class OpenTSDBClient(object): + """OpenTSDBClient primary client object to connect OpenTSDB. + The :class:`~.OpenTSDBClient` object holds information necessary to + connect to OpenTSDB. Requests can be made to OpenTSDB directly through + the client. + :param host: hostname to connect to OpenTSDB, defaults to 'localhost' + :type host: str + :param port: port to connect to OpenTSDB, defaults to 4242 + :type port: int + :param username: user to connect, defaults to 'root' + :type username: str + :param password: password of the user, defaults to 'root' + :type password: str + :param pool_size: urllib3 connection pool size, defaults to 10. + :type pool_size: int + :param ssl: use https instead of http to connect to OpenTSDB, defaults to + False + :type ssl: bool + :param verify_ssl: verify SSL certificates for HTTPS requests, defaults to + False + :type verify_ssl: bool + :param timeout: number of seconds Requests will wait for your client to + establish a connection, defaults to None + :type timeout: int + :param retries: number of retries your client will try before aborting, + defaults to 3. 0 indicates try until success + :type retries: int + :param proxies: HTTP(S) proxy to use for Requests, defaults to {} + :type proxies: dict + :param cert: Path to client certificate information to use for mutual TLS + authentication. You can specify a local cert to use + as a single file containing the private key and the certificate, or as + a tuple of both files’ paths, defaults to None + :type cert: str + :raises ValueError: if cert is provided but ssl is disabled (set to False) + """ + + def __init__( + self, + host='localhost', + port=4242, + username='root', + password='root', + ssl=False, + verify_ssl=False, + timeout=None, + retries=3, + proxies=None, + pool_size=10, + cert=None, + ): + """Construct a new OpenTSDBClient object.""" + self._host = host + self._port = int(port) + self._username = username + self._password = password + self._timeout = timeout + self._retries = retries + + self._verify_ssl = verify_ssl + + self._session = requests.Session() + adapter = requests.adapters.HTTPAdapter( + pool_connections=int(pool_size), pool_maxsize=int(pool_size)) + + self._scheme = "http" + + if ssl is True: + self._scheme = "https" + + self._session.mount(self._scheme + '://', adapter) + + if proxies is None: + self._proxies = {} + else: + self._proxies = proxies + + if cert: + if not ssl: + raise ValueError( + "Client certificate provided but ssl is disabled.") + else: + self._session.cert = cert + + self._baseurl = "{0}://{1}:{2}".format( + self._scheme, self._host, self._port) + + self._headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + + def write(self, data, expected_response_code=204): + """Write data to OpenTSDB. + :param data: the data to be written + :param expected_response_code: the expected response code of the write + operation, defaults to 204 + :type expected_response_code: int + :returns: True, if the write operation is successful + :rtype: bool + """ + headers = self._headers + headers['Content-Type'] = 'application/json' + + self.request( + url="api/put", + method='POST', + data=data, + expected_response_code=expected_response_code, + headers=headers) + return True + + def request( + self, + url, + method='GET', + params=None, + data=None, + expected_response_code=200, + headers=None): + """Make a HTTP request to the OpenTSDB API. + :param url: the path of the HTTP request + :type url: str + :param method: the HTTP method for the request, defaults to GET + :type method: str + :param params: additional parameters for the request, defaults to None + :type params: dict + :param data: the data of the request, defaults to None + :type data: str + :param expected_response_code: the expected response code of + the request, defaults to 200 + :type expected_response_code: int + :param headers: headers to add to the request + :type headers: dict + :returns: the response from the request + :rtype: :class:`requests.Response` + :raises OpenTSDBServerError: if the response code is any server error + code (5xx) + :raises OpenTSDBClientError: if the response code is not the + same as `expected_response_code` and is not a server error code + """ + url = "{0}/{1}".format(self._baseurl, url) + + if headers is None: + headers = self._headers + + if params is None: + params = {} + + if isinstance(data, (dict, list)): + data = json.dumps(data) + + # Try to send the request more than once by default (see #103) + retry = True + _try = 0 + while retry: + try: + response = self._session.request( + method=method, + url=url, + auth=(self._username, self._password), + params=params, + data=data, + headers=headers, + proxies=self._proxies, + verify=self._verify_ssl, + timeout=self._timeout) + break + except (requests.exceptions.ConnectionError, + requests.exceptions.HTTPError, requests.exceptions.Timeout): + _try += 1 + if self._retries != 0: + retry = _try < self._retries + if method == "POST": + time.sleep((2 ** _try) * random.random() / 100.0) + if not retry: + raise + # if there's not an error, there must have been a successful response + if 500 <= response.status_code < 600: + raise Exception(response.content) + elif response.status_code == expected_response_code: + return response + else: + raise Exception(response.content, response.status_code) diff --git a/yandextank/plugins/OpenTSDBUploader/config/schema.yaml b/yandextank/plugins/OpenTSDBUploader/config/schema.yaml new file mode 100644 index 000000000..3c6dc1fd1 --- /dev/null +++ b/yandextank/plugins/OpenTSDBUploader/config/schema.yaml @@ -0,0 +1,36 @@ +tank_tag: + default: unknown + type: string +address: + default: localhost + type: string +port: + default: 4242 + type: integer +username: + default: root + type: string +password: + default: root + type: string +chunk_size: + default: 4096 + type: integer +labeled: + default: false + type: boolean +histograms: + default: false + type: boolean +prefix_metric: + default: "" + type: string +custom_tags: + default: {} + type: dict +ssl: + default: true + type: boolean +verify_ssl: + default: true + type: boolean diff --git a/yandextank/plugins/OpenTSDBUploader/decoder.py b/yandextank/plugins/OpenTSDBUploader/decoder.py new file mode 100644 index 000000000..cdfeae424 --- /dev/null +++ b/yandextank/plugins/OpenTSDBUploader/decoder.py @@ -0,0 +1,177 @@ +import itertools +import time + + +def uts(dt): + return int(time.mktime(dt.timetuple())) + + +class Decoder(object): + """ + Decode metrics incoming from tank into points for OpenTSDB + + Parameters + ---------- + parent_tags : dict + common per-test tags + tank_tag : basestring + tank identifier tag + uuid : basestring + test id tag + labeled : bool + detailed stats for each label + histograms : bool + response time histograms measurements + """ + + def __init__(self, tank_tag, uuid, parent_tags, labeled, histograms): + self.labeled = labeled + initial_tags = {"tank": tank_tag, "uuid": uuid} + initial_tags.update(parent_tags) + self.tags = initial_tags + self.histograms = histograms + + def set_uuid(self, id_): + self.tags['uuid'] = id_ + + def decode_monitoring(self, data): + """ + The reason why we have two separate methods for monitoring + and aggregates is a strong difference in incoming data. + """ + points = list() + for second_data in data: + for host, host_data in second_data["data"].iteritems(): + points.append( + self.__make_points( + "monitoring", + { + "host": host, + "comment": host_data.get("comment") + }, + second_data["timestamp"], + { + # cast int to float. avoid + # https://github.com/yandex/yandex-tank/issues/776 + metric: + float(value) if isinstance( + value, int) else value + for metric, value in + host_data["metrics"].iteritems() + })) + return list(itertools.chain(*points)) + + def decode_aggregates(self, aggregated_data, gun_stats, prefix): + ts = aggregated_data["ts"] + points = list() + # stats overall w/ __OVERALL__ label + points += self.__make_points_for_label( + ts, aggregated_data["overall"], "__OVERALL__", prefix, gun_stats) + # detailed stats per tag + if self.labeled: + for label, aggregated_data_by_tag in aggregated_data[ + "tagged"].items(): + points += self.__make_points_for_label( + ts, aggregated_data_by_tag, label, prefix, gun_stats) + return points + + def __make_points_for_label(self, ts, data, label, prefix, gun_stats): + """x + Make a set of points for `this` label + + overall_quantiles, overall_meta, net_codes, proto_codes, histograms + """ + label_points = list() + + label_points.extend(( + # overall quantiles for label + self.__make_points( + prefix + "overall_quantiles", {"label": label}, ts, + self.__make_quantile_fields(data)), + # overall meta (gun status) for label + self.__make_points( + prefix + "overall_meta", {"label": label}, ts, + self.__make_overall_meta_fields(data, gun_stats)), + # net codes for label + self.__make_points( + prefix + "net_codes", {"label": label}, ts, + self.__make_netcodes_fields(data)), + # proto codes for label + self.__make_points( + prefix + "proto_codes", {"label": label}, ts, + self.__make_protocodes_fields(data)))) + # histograms, one row for each bin + if self.histograms: + for bin_, count in zip(data["interval_real"]["hist"]["bins"], + data["interval_real"]["hist"]["data"]): + label_points.append( + self.__make_points( + prefix + "histograms", {"label": label}, ts, { + "bin": bin_, + "count": count + })) + return list(itertools.chain(*label_points)) + + @staticmethod + def __make_quantile_fields(data): + return { + 'q' + str(q): value / 1000.0 + for q, value in zip( + data["interval_real"]["q"]["q"], data["interval_real"]["q"] + ["value"]) + } + + @staticmethod + def __make_overall_meta_fields(data, stats): + return { + "active_threads": stats["metrics"]["instances"], + "RPS": data["interval_real"]["len"], + "planned_requests": float(stats["metrics"]["reqps"]), + } + + @staticmethod + def __make_netcodes_fields(data): + return { + str(code): int(cnt) + for code, cnt in data["net_code"]["count"].items() + } + + @staticmethod + def __make_protocodes_fields(data): + return { + str(code): int(cnt) + for code, cnt in data["proto_code"]["count"].items() + } + + def __make_points(self, metric, additional_tags, ts, fields): + """ + Parameters + ---------- + metric : string + metric type (e.g. monitoring, overall_meta, net_codes, proto_codes, overall_quantiles) + additional_tags : dict + custom additional tags for this points + ts : integer + timestamp + fields : dict + opentsdb fields tag + + Returns + ------- + dict + points for OpenTSDB client + """ + result = [] + + for key, value in fields.items(): + tags = self.tags.copy() + tags.update(additional_tags) + tags["field"] = key + result.append({ + "metric": metric, + "tags": tags, + "timestamp": int(ts), + "value": value + }) + + return result diff --git a/yandextank/plugins/OpenTSDBUploader/plugin.py b/yandextank/plugins/OpenTSDBUploader/plugin.py new file mode 100644 index 000000000..4deef61e3 --- /dev/null +++ b/yandextank/plugins/OpenTSDBUploader/plugin.py @@ -0,0 +1,90 @@ +# coding=utf-8 +# TODO: make the next two lines unnecessary +# pylint: disable=line-too-long +# pylint: disable=missing-docstring +import datetime +import logging +import sys +from builtins import str +from uuid import uuid4 + +from .client import OpenTSDBClient +from .decoder import Decoder +from ...common.interfaces import AbstractPlugin, \ + MonitoringDataListener, AggregateResultListener + +logger = logging.getLogger(__name__) # pylint: disable=C0103 + + +def chop(data_list, chunk_size): + if sys.getsizeof(str(data_list)) <= chunk_size: + return [data_list] + elif len(data_list) == 1: + logger.warning( + "Too large piece of Telegraf data. Might experience upload problems." + ) + return [data_list] + else: + mid = len(data_list) / 2 + return chop(data_list[:mid], chunk_size) + chop( + data_list[mid:], chunk_size) + + +class Plugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener): + SECTION = 'opentsdb' + + def __init__(self, core, cfg, name): + AbstractPlugin.__init__(self, core, cfg, name) + self.tank_tag = self.get_option("tank_tag") + self.prefix_metric = self.get_option("prefix_metric") + self._client = None + self.start_time = None + self.end_time = None + + self.decoder = Decoder( + self.tank_tag, + str(uuid4()), + self.get_option("custom_tags"), + self.get_option("labeled"), + self.get_option("histograms"), + ) + + @property + def client(self): + if not self._client: + self._client = OpenTSDBClient( + host=self.get_option("address"), + port=self.get_option("port"), + username=self.get_option("username"), + password=self.get_option("password"), + ssl=self.get_option("ssl"), + verify_ssl=self.get_option("verify_ssl") + ) + return self._client + + def prepare_test(self): + self.core.job.subscribe_plugin(self) + + def start_test(self): + self.start_time = datetime.datetime.now() + + def end_test(self, retcode): + self.end_time = datetime.datetime.now() + datetime.timedelta(minutes=1) + return retcode + + def on_aggregated_data(self, data, stats): + self.client.write( + self.decoder.decode_aggregates(data, stats, self.prefix_metric)) + + def monitoring_data(self, data_list): + if len(data_list) > 0: + [ + self._send_monitoring(chunk) + for chunk in chop(data_list, self.get_option("chunk_size")) + ] + + def _send_monitoring(self, data): + self.client.write(self.decoder.decode_monitoring(data)) + + def set_uuid(self, id_): + self.decoder.tags['uuid'] = id_ diff --git a/yandextank/plugins/OpenTSDBUploader/tests/test_opentsdb_decoder.py b/yandextank/plugins/OpenTSDBUploader/tests/test_opentsdb_decoder.py new file mode 100644 index 000000000..bc01af3dc --- /dev/null +++ b/yandextank/plugins/OpenTSDBUploader/tests/test_opentsdb_decoder.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +from uuid import uuid4 + +from yandextank.plugins.OpenTSDBUploader.decoder import Decoder + + +class TestDecoder(object): + def test_metrics_cast(self): + test_uuid = str(uuid4()) + tank_tag = 'test_tank_tag' + comment = 'test comment' + raw_metrics = { + 'metric1': -123, + 'metric2': -123.456, + 'metric3': 123, + 'metric4': 123.456, + 'metric5': 0, + 'metric6': -0.1, + 'metric7': 0.1, + 'metric8': 'down', + } + timestamp = 123456789 + host = '127.0.0.1' + data = [{ + 'data': { + host: { + 'comment': comment, + 'metrics': raw_metrics + } + }, + 'timestamp': timestamp + }] + expected_metrics = { + 'metric1': -123.0, + 'metric2': -123.456, + 'metric3': 123.0, + 'metric4': 123.456, + 'metric5': 0.0, + 'metric6': -0.1, + 'metric7': 0.1, + 'metric8': 'down' + } + + decoder = Decoder(tank_tag, test_uuid, {}, True, True) + result_points = decoder.decode_monitoring(data) + + assert (len(result_points) == len(expected_metrics)) + # check other props + for r_point in result_points: + assert (r_point['timestamp'] == timestamp) + assert (r_point['metric'] == 'monitoring') + assert (r_point['tags']['comment'] == comment) + assert (r_point['tags']['host'] == host) + assert (r_point['tags']['tank'] == tank_tag) + assert (r_point['tags']['uuid'] == test_uuid) + if r_point['tags']['field'] not in expected_metrics: + assert False + if not isinstance(r_point['value'], type( + expected_metrics[r_point['tags']['field']])): + assert False + if not r_point['value'] == expected_metrics[r_point['tags'] + ['field']]: + assert False From b758f062e50646ee9f45a8c754cbd19088933f9e Mon Sep 17 00:00:00 2001 From: fomars Date: Fri, 22 Nov 2019 16:56:32 +0300 Subject: [PATCH 25/28] netort 0.7.3, thread safe property, docs --- docs/conf.py | 4 +-- docs/config_reference.rst | 33 ++++++++++++++++--------- setup.py | 2 +- yandextank/plugins/NeUploader/plugin.py | 10 +++----- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index bb5f313f6..133aff7ac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,9 +52,9 @@ # built documents. # # The short X.Y version. -version = '1.11' +version = '1.12' # The full version, including alpha/beta/rc tags. -release = '1.11.2' +release = '1.12.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/config_reference.rst b/docs/config_reference.rst index af0f01418..81b2cfbd6 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -64,8 +64,8 @@ ShootExec :nullable: True -Influx -====== +InfluxUploader +============== ``address`` (string) -------------------- @@ -83,13 +83,9 @@ Influx --------------------- *\- (no description). Default:* ``mydb`` -``grafana_dashboard`` (string) ------------------------------- -*\- (no description). Default:* ``tank-dashboard`` - -``grafana_root`` (string) -------------------------- -*\- (no description). Default:* ``http://localhost/`` +``histograms`` (boolean) +------------------------ +*\- (no description). Default:* ``False`` ``labeled`` (boolean) --------------------- @@ -266,11 +262,26 @@ Pandora ``expvar`` (boolean) -------------------- -*\- Toggle expvar monitoring. Default:* ``True`` +*\- (no description). Default:* ``False`` ``pandora_cmd`` (string) ------------------------ -*\- Pandora executable path. Default:* ``pandora`` +*\- Pandora executable path or link to it. Default:* ``pandora`` + +``report_file`` (string) +------------------------ +*\- Pandora phout path (normally will be taken from pandora config). Default:* ``None`` + +:nullable: + True + +``resource`` (dict) +------------------- +*\- dict with attributes for additional resources.* + +``resources`` (list) +-------------------- +*\- additional resources you need to download before test. Default:* ``[]`` Android ======= diff --git a/setup.py b/setup.py index a2a059e29..355576c64 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ 'psutil>=1.2.1', 'requests>=2.5.1', 'paramiko>=1.16.0', 'pandas==0.24.2', 'numpy==1.15.4', 'future>=0.16.0', 'pip>=8.1.2', - 'pyyaml>=4.2b1', 'cerberus==1.3.1', 'influxdb>=5.0.0', 'netort>=0.7.1', + 'pyyaml>=4.2b1', 'cerberus==1.3.1', 'influxdb>=5.0.0', 'netort>=0.7.3', 'retrying>=1.3.3', 'pytest-runner', 'typing' ], setup_requires=[ diff --git a/yandextank/plugins/NeUploader/plugin.py b/yandextank/plugins/NeUploader/plugin.py index 6bb0c8b79..b53835145 100644 --- a/yandextank/plugins/NeUploader/plugin.py +++ b/yandextank/plugins/NeUploader/plugin.py @@ -1,7 +1,7 @@ import logging import pandas -from netort.data_manager import DataSession +from netort.data_manager import DataSession, thread_safe_property from yandextank.plugins.Phantom.reader import string_to_df_microsec from yandextank.common.interfaces import AbstractPlugin,\ @@ -40,10 +40,9 @@ def start_test(self): self.is_test_finished = lambda: -1 self.reader = [] - @property + @thread_safe_property def col_map(self): - if self._col_map is None: - self._col_map = { + return { 'interval_real': self.data_session.new_true_metric, 'connect_time': self.data_session.new_true_metric, 'send_time': self.data_session.new_true_metric, @@ -53,9 +52,8 @@ def col_map(self): 'net_code': self.data_session.new_event_metric, 'proto_code': self.data_session.new_event_metric } - return self._col_map - @property + @thread_safe_property def data_session(self): """ :rtype: DataSession From 0b06d82b2c561e7c199053de6372e4865a6de58a Mon Sep 17 00:00:00 2001 From: Ian Duffy Date: Fri, 22 Nov 2019 22:06:46 +0000 Subject: [PATCH 26/28] Expose more metrics over opentsdb --- .../plugins/OpenTSDBUploader/client/client.py | 3 +- .../plugins/OpenTSDBUploader/decoder.py | 45 ++++++++++++------- yandextank/plugins/OpenTSDBUploader/plugin.py | 3 +- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/yandextank/plugins/OpenTSDBUploader/client/client.py b/yandextank/plugins/OpenTSDBUploader/client/client.py index 2b94e2e6a..77d107410 100644 --- a/yandextank/plugins/OpenTSDBUploader/client/client.py +++ b/yandextank/plugins/OpenTSDBUploader/client/client.py @@ -44,7 +44,6 @@ class OpenTSDBClient(object): :type cert: str :raises ValueError: if cert is provided but ssl is disabled (set to False) """ - def __init__( self, host='localhost', @@ -182,7 +181,7 @@ def request( if self._retries != 0: retry = _try < self._retries if method == "POST": - time.sleep((2 ** _try) * random.random() / 100.0) + time.sleep((2**_try) * random.random() / 100.0) if not retry: raise # if there's not an error, there must have been a successful response diff --git a/yandextank/plugins/OpenTSDBUploader/decoder.py b/yandextank/plugins/OpenTSDBUploader/decoder.py index cdfeae424..248adf272 100644 --- a/yandextank/plugins/OpenTSDBUploader/decoder.py +++ b/yandextank/plugins/OpenTSDBUploader/decoder.py @@ -1,5 +1,6 @@ import itertools import time +from ...common import util def uts(dt): @@ -23,7 +24,6 @@ class Decoder(object): histograms : bool response time histograms measurements """ - def __init__(self, tank_tag, uuid, parent_tags, labeled, histograms): self.labeled = labeled initial_tags = {"tank": tank_tag, "uuid": uuid} @@ -54,8 +54,7 @@ def decode_monitoring(self, data): # cast int to float. avoid # https://github.com/yandex/yandex-tank/issues/776 metric: - float(value) if isinstance( - value, int) else value + float(value) if isinstance(value, int) else value for metric, value in host_data["metrics"].iteritems() })) @@ -94,12 +93,16 @@ def __make_points_for_label(self, ts, data, label, prefix, gun_stats): self.__make_overall_meta_fields(data, gun_stats)), # net codes for label self.__make_points( - prefix + "net_codes", {"label": label}, ts, - self.__make_netcodes_fields(data)), + prefix + "net_codes", {"label": label}, + ts, + self.__make_netcodes_fields(data), + field_lookup_table=util.NET), # proto codes for label self.__make_points( - prefix + "proto_codes", {"label": label}, ts, - self.__make_protocodes_fields(data)))) + prefix + "proto_codes", {"label": label}, + ts, + self.__make_protocodes_fields(data), + field_lookup_table=util.HTTP))) # histograms, one row for each bin if self.histograms: for bin_, count in zip(data["interval_real"]["hist"]["bins"], @@ -124,26 +127,36 @@ def __make_quantile_fields(data): @staticmethod def __make_overall_meta_fields(data, stats): return { - "active_threads": stats["metrics"]["instances"], - "RPS": data["interval_real"]["len"], - "planned_requests": float(stats["metrics"]["reqps"]), + "active_threads": + stats["metrics"]["instances"], + "RPS": + data["interval_real"]["len"], + "planned_requests": + float(stats["metrics"]["reqps"]), + "avg_rt": + float(data['interval_real']['total']) / data['interval_real']['len'] / 1000.0, + "min": + data['interval_real']['min'] / 1000.0, + "max": + data['interval_real']['max'] / 1000.0 } @staticmethod def __make_netcodes_fields(data): return { - str(code): int(cnt) + int(code): int(cnt) for code, cnt in data["net_code"]["count"].items() } @staticmethod def __make_protocodes_fields(data): return { - str(code): int(cnt) + int(code): int(cnt) for code, cnt in data["proto_code"]["count"].items() } - def __make_points(self, metric, additional_tags, ts, fields): + def __make_points( + self, metric, additional_tags, ts, fields, field_lookup_table={}): """ Parameters ---------- @@ -166,12 +179,14 @@ def __make_points(self, metric, additional_tags, ts, fields): for key, value in fields.items(): tags = self.tags.copy() tags.update(additional_tags) - tags["field"] = key + tags["field"] = str(key) + if field_lookup_table.get(key): + tags["field_label"] = field_lookup_table.get(key).replace(" ", "_") result.append({ "metric": metric, "tags": tags, "timestamp": int(ts), - "value": value + "value": value, }) return result diff --git a/yandextank/plugins/OpenTSDBUploader/plugin.py b/yandextank/plugins/OpenTSDBUploader/plugin.py index 4deef61e3..28fcd918a 100644 --- a/yandextank/plugins/OpenTSDBUploader/plugin.py +++ b/yandextank/plugins/OpenTSDBUploader/plugin.py @@ -58,8 +58,7 @@ def client(self): username=self.get_option("username"), password=self.get_option("password"), ssl=self.get_option("ssl"), - verify_ssl=self.get_option("verify_ssl") - ) + verify_ssl=self.get_option("verify_ssl")) return self._client def prepare_test(self): From 7e060e7fdb563ed39bce6f7fd50d76fe04d0dd51 Mon Sep 17 00:00:00 2001 From: fomars Date: Mon, 25 Nov 2019 19:07:28 +0300 Subject: [PATCH 27/28] netort up --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 355576c64..a837b3cd3 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ 'psutil>=1.2.1', 'requests>=2.5.1', 'paramiko>=1.16.0', 'pandas==0.24.2', 'numpy==1.15.4', 'future>=0.16.0', 'pip>=8.1.2', - 'pyyaml>=4.2b1', 'cerberus==1.3.1', 'influxdb>=5.0.0', 'netort>=0.7.3', + 'pyyaml>=4.2b1', 'cerberus==1.3.1', 'influxdb>=5.0.0', 'netort>=0.7.4', 'retrying>=1.3.3', 'pytest-runner', 'typing' ], setup_requires=[ From 189ef81a2fa9136069d4f24a52a86d53707ce38d Mon Sep 17 00:00:00 2001 From: fomars Date: Mon, 25 Nov 2019 22:14:15 +0300 Subject: [PATCH 28/28] pep8 --- yandextank/plugins/NeUploader/plugin.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/yandextank/plugins/NeUploader/plugin.py b/yandextank/plugins/NeUploader/plugin.py index b53835145..23e4646d4 100644 --- a/yandextank/plugins/NeUploader/plugin.py +++ b/yandextank/plugins/NeUploader/plugin.py @@ -43,15 +43,15 @@ def start_test(self): @thread_safe_property def col_map(self): return { - 'interval_real': self.data_session.new_true_metric, - 'connect_time': self.data_session.new_true_metric, - 'send_time': self.data_session.new_true_metric, - 'latency': self.data_session.new_true_metric, - 'receive_time': self.data_session.new_true_metric, - 'interval_event': self.data_session.new_true_metric, - 'net_code': self.data_session.new_event_metric, - 'proto_code': self.data_session.new_event_metric - } + 'interval_real': self.data_session.new_true_metric, + 'connect_time': self.data_session.new_true_metric, + 'send_time': self.data_session.new_true_metric, + 'latency': self.data_session.new_true_metric, + 'receive_time': self.data_session.new_true_metric, + 'interval_event': self.data_session.new_true_metric, + 'net_code': self.data_session.new_event_metric, + 'proto_code': self.data_session.new_event_metric + } @thread_safe_property def data_session(self):