From 54b04f206908158b2b8963d41a801f4598917907 Mon Sep 17 00:00:00 2001 From: ival83 Date: Fri, 20 Sep 2019 18:12:07 +0300 Subject: [PATCH 1/6] support of http source for test resources --- yandextank/plugins/Pandora/config/schema.yaml | 9 +++++- yandextank/plugins/Pandora/plugin.py | 32 ++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/yandextank/plugins/Pandora/config/schema.yaml b/yandextank/plugins/Pandora/config/schema.yaml index 3a32d0e2c..fc1b1a5a4 100644 --- a/yandextank/plugins/Pandora/config/schema.yaml +++ b/yandextank/plugins/Pandora/config/schema.yaml @@ -6,7 +6,7 @@ affinity: pandora_cmd: type: string default: pandora - description: Pandora executable path + description: Pandora executable path or link to it buffered_seconds: type: integer default: 2 @@ -21,6 +21,13 @@ config_file: expvar: type: boolean default: false +resources: + default: [] + type: list + description: additional resources you need to download befor test +resource: + type: dict + description: dict with attributes for additional resources report_file: type: string nullable: true diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 8abfbe75e..a93083313 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -2,6 +2,7 @@ import logging import subprocess import time +import os from threading import Event import yaml @@ -43,6 +44,7 @@ def __init__(self, core, cfg, name): self.__schedule = None self.ammofile = None self.process_stderr_file = None + self.resources = [] @staticmethod def get_key(): @@ -56,11 +58,15 @@ def get_available_options(self): return opts def configure(self): - self.pandora_cmd = self.get_option("pandora_cmd") + #self.pandora_cmd = self.get_option("pandora_cmd") self.report_file = self.get_option("report_file") self.buffered_seconds = self.get_option("buffered_seconds") self.affinity = self.get_option("affinity", "") + self.resources = self.get_option("resources") + #if we use custom pandora binary, we can download it and make it executable + self.pandora_cmd = self.get_resource(self.get_option("pandora_cmd"), "./pandora", permissions=0755) + # get config_contents and patch it: expand resources via resource manager # config_content option has more priority over config_file if self.get_option("config_content"): @@ -75,6 +81,11 @@ def configure(self): raise RuntimeError("Neither pandora.config_content, nor pandora.config_file specified") logger.debug('Config after parsing for patching: %s', self.config_contents) + #download all resources from self.get_options("resources") + if len(self.resources) > 0: + for resource in self.resources: + self.get_resource(resource["src"],resource["dst"]) + # find report filename and add to artifacts self.report_file = self.__find_report_filename() with open(self.report_file, 'w'): @@ -186,6 +197,24 @@ def get_stats_reader(self): self.stats_reader = PandoraStatsReader(self.expvar_enabled, self.expvar_port) return self.stats_reader + def get_resource(self, resource, dst, permissions=0644): + opener = resource_manager.get_opener(resource) + if isinstance(opener, HttpOpener): + tmp_path = opener.download_file(True, try_ungzip=True) + try: + os.rename(tmp_path, dst) + logger.info('Successfully moved resource %s', dst) + except OSError as ex: + logger.debug("Could not move resource %s\n%s", dst, ex) + else: + dst = opener.get_filename + try: + os.chmod(dst, permissions) + logger.info('Permissions on %s have changed %d', dst, permissions) + except OSError as ex: + logger.debug("Could not chane pepermissions on %s\n%s", dst, ex) + return dst + def prepare_test(self): try: console = self.core.get_plugin_of_type(ConsolePlugin) @@ -199,6 +228,7 @@ def prepare_test(self): self.core.job.aggregator.add_result_listener(widget) def start_test(self): + print self.pandora_cmd args = [self.pandora_cmd] +\ (['-expvar'] if self.expvar else []) +\ [self.pandora_config_file] From 291cc614833af1def785df281f6a9278bb2ade8d Mon Sep 17 00:00:00 2001 From: ival83 Date: Fri, 20 Sep 2019 18:16:22 +0300 Subject: [PATCH 2/6] remove comment --- yandextank/plugins/Pandora/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index a93083313..511ecc66a 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -58,7 +58,6 @@ def get_available_options(self): return opts def configure(self): - #self.pandora_cmd = self.get_option("pandora_cmd") self.report_file = self.get_option("report_file") self.buffered_seconds = self.get_option("buffered_seconds") self.affinity = self.get_option("affinity", "") From 1b381e03ac2d3dfd969b344b6b8f9327417fcf30 Mon Sep 17 00:00:00 2001 From: ival83 Date: Fri, 20 Sep 2019 19:45:03 +0300 Subject: [PATCH 3/6] code style --- yandextank/plugins/Pandora/plugin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 511ecc66a..2edcb60ee 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -63,9 +63,9 @@ def configure(self): self.affinity = self.get_option("affinity", "") self.resources = self.get_option("resources") - #if we use custom pandora binary, we can download it and make it executable + # if we use custom pandora binary, we can download it and make it executable self.pandora_cmd = self.get_resource(self.get_option("pandora_cmd"), "./pandora", permissions=0755) - + # get config_contents and patch it: expand resources via resource manager # config_content option has more priority over config_file if self.get_option("config_content"): @@ -80,10 +80,10 @@ def configure(self): raise RuntimeError("Neither pandora.config_content, nor pandora.config_file specified") logger.debug('Config after parsing for patching: %s', self.config_contents) - #download all resources from self.get_options("resources") + # download all resources from self.get_options("resources") if len(self.resources) > 0: for resource in self.resources: - self.get_resource(resource["src"],resource["dst"]) + self.get_resource(resource["src"], resource["dst"]) # find report filename and add to artifacts self.report_file = self.__find_report_filename() From 6cf3749cd68f94cdc637943f36002def43de0935 Mon Sep 17 00:00:00 2001 From: ival83 Date: Fri, 20 Sep 2019 19:50:56 +0300 Subject: [PATCH 4/6] debug prints removed --- yandextank/plugins/Pandora/config/schema.yaml | 2 +- yandextank/plugins/Pandora/plugin.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/yandextank/plugins/Pandora/config/schema.yaml b/yandextank/plugins/Pandora/config/schema.yaml index fc1b1a5a4..d9a0012f9 100644 --- a/yandextank/plugins/Pandora/config/schema.yaml +++ b/yandextank/plugins/Pandora/config/schema.yaml @@ -24,7 +24,7 @@ expvar: resources: default: [] type: list - description: additional resources you need to download befor test + description: additional resources you need to download before test resource: type: dict description: dict with attributes for additional resources diff --git a/yandextank/plugins/Pandora/plugin.py b/yandextank/plugins/Pandora/plugin.py index 2edcb60ee..cdf57e15c 100644 --- a/yandextank/plugins/Pandora/plugin.py +++ b/yandextank/plugins/Pandora/plugin.py @@ -227,7 +227,6 @@ def prepare_test(self): self.core.job.aggregator.add_result_listener(widget) def start_test(self): - print self.pandora_cmd args = [self.pandora_cmd] +\ (['-expvar'] if self.expvar else []) +\ [self.pandora_config_file] From b763da6845cd86c8d4440b34609d0b2f0b3ede42 Mon Sep 17 00:00:00 2001 From: Mikhail Dyomin Date: Wed, 25 Sep 2019 18:29:29 +0300 Subject: [PATCH 5/6] Fix _stop_agent() for localhost --- yandextank/plugins/Telegraf/client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/yandextank/plugins/Telegraf/client.py b/yandextank/plugins/Telegraf/client.py index b4ca92502..aa5a0b310 100644 --- a/yandextank/plugins/Telegraf/client.py +++ b/yandextank/plugins/Telegraf/client.py @@ -138,13 +138,16 @@ def read_buffer(self): 'otherwise something really nasty happend', exc_info=True) + def _stop_agent(self): + if self.session: + self.session.terminate() + logger.info('Waiting localhost agent to terminate...') + def uninstall(self): """ Remove agent's files from remote host """ if self.session: - logger.info('Waiting monitoring data...') - self.session.terminate() self.session.wait() self.session = None log_filename = "agent_{host}.log".format(host="localhost") From 5c4d48cdda75c80ad4c6feadf81fcf3a4d52004f Mon Sep 17 00:00:00 2001 From: Nurlan Date: Wed, 25 Sep 2019 20:46:04 +0300 Subject: [PATCH 6/6] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e02528958..3b1785239 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='yandextank', - version='1.12.5', + version='1.12.5.1', description='a performance measurement tool', longer_description=''' Yandex.Tank is a performance measurement and load testing automatization tool.