Merge pull request #434 from yandex/release-1.9.1

Release 1.9.1

docs/core_and_modules.rst

@@ -752,7 +752,7 @@ Disable phantom first (unless you really want to keep it active alongside at you
           "AmmoLimit": 10000000
         },
         "result": {
-          "type": "log/phout",
+          "type": "phout",
           "destination": "./phout.log"
         },
         "shared-limits": false,

mocks/README.md

@@ -0,0 +1 @@
+Mocks to test & develop various tank components

mocks/shootexec-shooter.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+import sys
+import time
+import random
+
+stdout = open(sys.argv[1], "w")
+stderr = open(sys.argv[2], "w")
+
+waitfor = time.time() + 60 * 2
+fake_rps = 1
+while time.time() < waitfor:
+    # shooting results
+    output = [
+        time.time(),
+        random.choice(["tag1", "tag2", "tag3"]),
+        int(500 * random.random()),
+        10,
+        10,
+        int(400 * random.random()),
+        10,
+        0,
+        int(1024 * random.random()),
+        int(1024 * random.random()),
+        0,
+        random.choice([200, 404, 503])
+    ]
+    stdout.write("\t".join([str(x) for x in output]) + "\n")
+    stdout.flush()
+
+    # shooter stats
+    stats = [
+        time.time(),
+        fake_rps,
+        1
+    ]
+    stderr.write("\t".join([str(x) for x in stats]) + "\n")
+    stderr.flush()
+
+    fake_rps += 100
+    time.sleep(0.3)
+
+sys.exit(0)

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='yandextank',
-    version='1.9.0',
+    version='1.9.1',
     description='a performance measurement tool',
     longer_description='''
 Yandex.Tank is a performance measurement and load testing automatization tool.
@@ -71,6 +71,7 @@
         'yandextank.plugins.RCAssert': ['config/*'],
         'yandextank.plugins.ResourceCheck': ['config/*'],
         'yandextank.plugins.ShellExec': ['config/*'],
+        'yandextank.plugins.ShootExec': ['config/*'],
         'yandextank.plugins.Telegraf': ['config/*'],
         'yandextank.plugins.TipsAndTricks': ['config/*'],
     },

yandextank/common/tests/test_util.py

@@ -1,5 +1,5 @@
 from queue import Queue
-from yandextank.common.util import Drain, Chopper
+from yandextank.common.util import Drain, Chopper, FileScanner
 
 
 class TestDrain(object):
@@ -41,3 +41,34 @@ def test_output(self):
         source = (range(i) for i in range(5))
         expected = [0, 0, 1, 0, 1, 2, 0, 1, 2, 3]
         assert list(Chopper(source)) == expected
+
+
+class TestFileScanner(object):
+    @staticmethod
+    def __process_chunks(chunks, sep="\n"):
+        reader = FileScanner("somefile.txt", sep=sep)
+        result = []
+        for chunk in chunks:
+            result.extend(reader._read_lines(chunk))
+        return result
+
+    def test_empty(self):
+        assert self.__process_chunks([""]) == []
+
+    def test_simple(self):
+        assert self.__process_chunks(["aaa\n", "bbb\n", "ccc\n"]) == ["aaa", "bbb", "ccc"]
+
+    def test_split(self):
+        assert self.__process_chunks(["aaa\nbbb\n", "ccc\n"]) == ["aaa", "bbb", "ccc"]
+
+    def test_join(self):
+        assert self.__process_chunks(["aaa", "bbb\n", "ccc\n"]) == ["aaabbb", "ccc"]
+
+    def test_no_first_separator(self):
+        assert self.__process_chunks(["aaa"]) == []
+
+    def test_no_last_separator(self):
+        assert self.__process_chunks(["aaa\n", "bbb\n", "ccc"]) == ["aaa", "bbb"]
+
+    def test_use_custom_separator(self):
+        assert self.__process_chunks(["aaa:bbb:ccc:"], ":") == ["aaa", "bbb", "ccc"]

yandextank/common/util.py

@@ -608,3 +608,39 @@ def recursive_dict_update(d1, d2):
         else:
             d1[k] = d2[k]
     return d1
+
+
+class FileScanner(object):
+    """
+    Basic class for stats reader for continiuos reading file line by line
+
+    Default line separator is a newline symbol. You can specify other separator
+    via constructor argument
+    """
+
+    _BUFSIZE = 4096
+
+    def __init__(self, path, sep="\n"):
+        self.__path = path
+        self.__sep = sep
+        self.__closed = False
+        self.__buffer = ""
+
+    def _read_lines(self, chunk):
+        self.__buffer += chunk
+        portions = self.__buffer.split(self.__sep)
+        for portion in portions[:-1]:
+            yield portion
+        self.__buffer = portions[-1]
+
+    def _read_data(self, lines):
+        raise NotImplementedError()
+
+    def __iter__(self):
+        with open(self.__path) as stats_file:
+            while not self.__closed:
+                chunk = stats_file.read(self._BUFSIZE)
+                yield self._read_data(self._read_lines(chunk))
+
+    def close(self):
+        self.__closed = True

yandextank/config_converter/converter.py

@@ -41,6 +41,7 @@ def parse_package_name(package_path):
     'JMeter': 'jmeter',
     'ResourceCheck': 'rcheck',
     'ShellExec': 'shell_?exec',
+    'ShootExec': 'shoot_?exec',
     'Console': 'console',
     'TipsAndTricks': 'tips',
     'RCAssert': 'rcassert',
@@ -393,6 +394,7 @@ def _guess_section_name(self):
             'RCAssert': 'rcassert',
             'ResourceCheck': 'rcheck',
             'ShellExec': 'shellexec',
+            'ShootExec': 'shootexec',
             'SvgReport': 'svgreport',
             'Telegraf': 'telegraf',
             'TipsAndTricks': 'tips'

yandextank/core/config/plugins_schema.yaml

@@ -1,9 +1,11 @@
 allow_unknown: true
 schema:
   enabled:
+    description: enable/disable the execution of the plugin
     type: boolean
     required: true
   package:
+    description: plugin\'s python package
     empty: false
     regex: '[^/]+'
     required: true

yandextank/core/config/schema.yaml

@@ -3,18 +3,18 @@ core:
   allow_unknown: false
   schema:
     affinity:
-      description: bind tank process to a specific cpu core(s)
+      description: specify cpu core(s) to bind tank process to
       type: string
       default: ''
     api_jobno:
-      description: tankapi job id, also used as test\'s directory name
+      description: tankapi job id, also used as test\'s directory name - determined by tank
       type: string
     artifacts_base_dir:
-#      description: base directory to store tests\' artifacts directories
+      description: base directory to store tests\' artifacts directories
       type: string
       default: ./logs
     artifacts_dir:
-#      description: directory inside base directory to store test\'s artifacts
+      description: directory inside base directory to store test\'s artifacts, defaults to api_jobno if null
       type: string
       nullable: true
       default: null
@@ -23,8 +23,10 @@ core:
     exitcode:
       type: integer
     flush_config_to:
+      description: path to store config
       type: string
     ignore_locks:
+      description: if tank is locked (*.lock file(s) presented in lock_dir), shoot nevertheless
       type: boolean
     uuid:
       type: string
@@ -33,9 +35,11 @@ core:
     message:
       type: string
     lock_dir:
+      description: directory to store *.lock files
       type: string
       default: /var/lock/
     operator:
+      description: your username
       type: string
     taskset_path:
       type: string

yandextank/core/consoleworker.py

@@ -1,5 +1,5 @@
 """ Provides classes to run TankCore from console environment """
-from ConfigParser import ConfigParser, MissingSectionHeaderError, NoOptionError
+from ConfigParser import ConfigParser, MissingSectionHeaderError, NoOptionError, NoSectionError
 import datetime
 import fnmatch
 import glob
@@ -84,12 +84,11 @@ def load_cfg(cfg_filename):
 
     :type cfg_filename: str
     """
-    if cfg_filename.endswith('.yaml'):
-        with open(cfg_filename) as f:
-            cfg = yaml.load(f)
+    if is_ini(cfg_filename):
+        return convert_ini(cfg_filename)
     else:
-        cfg = convert_ini(cfg_filename)
-    return cfg
+        with open(cfg_filename) as f:
+            return yaml.load(f)
 
 
 def cfg_folder_loader(path):
@@ -216,8 +215,6 @@ def patch_ini_config_with_monitoring(ini_config, mon_section_name):
             :type ini_config: ConfigParser
             """
             CONFIG = 'config'
-            if not ini_config.has_section(mon_section_name):
-                raise NoOptionError
             telegraf_cfg = ini_config.get(mon_section_name, CONFIG)
             if not telegraf_cfg.startswith('<') and not telegraf_cfg.lower() == 'auto':
                 with open(resource_manager.resource_filename(telegraf_cfg), 'rb') as telegraf_cfg_file:
@@ -227,10 +224,10 @@ def patch_ini_config_with_monitoring(ini_config, mon_section_name):
 
         try:
             cfg_ini = patch_ini_config_with_monitoring(cfg_ini, 'monitoring')
-        except NoOptionError:
+        except (NoSectionError, NoOptionError):
             try:
                 patch_ini_config_with_monitoring(cfg_ini, 'telegraf')
-            except NoOptionError:
+            except (NoOptionError, NoSectionError):
                 pass
 
         for section, key, value in depr_options:
@@ -278,7 +275,11 @@ def __init__(self, options, ammofile):
                 'ammofile': ammofile
             }
 
-        self.core = load_tank_core(options.config, options.option, options.no_rc, [], overwrite_options)
+        self.core = load_tank_core([resource_manager.resource_filename(cfg) for cfg in options.config],
+                                   options.option,
+                                   options.no_rc,
+                                   [],
+                                   overwrite_options)
 
         raw_cfg_file, raw_cfg_path = tempfile.mkstemp(suffix='_pre-validation-config.yaml')
         os.close(raw_cfg_file)

yandextank/plugins/Autostop/config/schema.yaml

@@ -1,8 +1,10 @@
 autostop:
+  description: list of autostop constraints
   type: list
   schema:
     type: string
   default: []
 report_file:
+  description: path to file to store autostop report
   type: string
   default: autostop_report.txt

yandextank/plugins/DataUploader/client.py

@@ -39,6 +39,12 @@ def __init__(
         self.session.verify = False
         self.session.headers.update({"User-Agent": "tank"})
 
+        if "https" in requests.utils.getproxies():
+            logger.info("Connecting via proxy %s" % requests.utils.getproxies()['https'])
+            self.session.proxies = requests.utils.getproxies()
+        else:
+            logger.info("Proxy not set")
+
         self.network_attempts = network_attempts
         self.network_timeout = network_timeout
         self.api_attempts = api_attempts

yandextank/plugins/DataUploader/plugin.py

@@ -276,6 +276,7 @@ def start_test(self):
             logger.debug("Saving jobno to: %s", jobno_file)
             with open(jobno_file, 'w') as fdes:
                 fdes.write(str(self.lp_job.number))
+            self.core.add_artifact_file(jobno_file)
         self.__save_conf()
 
     def is_test_finished(self):

yandextank/plugins/JMeter/plugin.py

@@ -168,9 +168,17 @@ def __add_jmeter_components(self, jmx, jtl, variables):
             source_lines = src_jmx.readlines()
 
         try:
+            # In new Jmeter version (3.2 as example) WorkBench's plugin checkbox enabled by default
+            # It totally crashes Yandex tank injection and raises XML Parse Exception
             closing = source_lines.pop(-1)
-            closing = source_lines.pop(-1) + closing
-            closing = source_lines.pop(-1) + closing
+            if "WorkBenchGui" in source_lines[-5]:
+                logger.info("WorkBench checkbox enabled...bypassing")
+                last_string_count = 6
+            else:
+                last_string_count = 2
+            while last_string_count > 0:
+                closing = source_lines.pop(-1) + closing
+                last_string_count -= 1
             logger.debug("Closing statement: %s", closing)
         except Exception as exc:
             raise RuntimeError("Failed to find the end of JMX XML: %s" % exc)

yandextank/plugins/ShootExec/__init__.py

@@ -0,0 +1 @@
+from .plugin import Plugin  # noqa: F401

yandextank/plugins/ShootExec/config/schema.yaml

@@ -0,0 +1,9 @@
+cmd:
+  type: string
+  required: true
+output_path:
+  type: string
+  required: true
+stats_path:
+  type: string
+  default: ''