Merge pull request #638 from yandex/release

Release 1.10.2

.travis.yml

@@ -4,7 +4,8 @@ python:
   - "2.7"
 install:
   - "pip install --upgrade setuptools"
+  - "pip install flake8"
   - "pip install ."
 script:
-  - "python setup.py flake8"
+  - "flake8 ."
   - "pytest -s"

docs/conf.py

@@ -52,9 +52,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.9'
+version = '1.10'
 # The full version, including alpha/beta/rc tags.
-release = '1.9.11'
+release = '1.10.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/config_reference.rst

@@ -3,6 +3,52 @@ Config reference
 ================
 
 
+Core
+====
+
+``core`` (dict)
+---------------
+*\- (no description).*
+
+:``affinity`` (string):
+ *\- specify cpu core(s) to bind tank process to,  http://linuxhowtos.org/manpages/1/taskset.htm. Default:* ``""``
+:``api_jobno`` (string):
+ *\- tankapi job id, also used as test\'s directory name \- determined by tank.*
+:``artifacts_base_dir`` (string):
+ *\- base directory to store tests\' artifacts directories. Default:* ``./logs``
+:``artifacts_dir`` (string):
+ *\- directory inside base directory to store test\'s artifacts, defaults to api_jobno if null. Default:* ``None``
+
+ :nullable:
+  True
+:``cmdline`` (string):
+ *\- (no description).*
+:``exitcode`` (integer):
+ *\- (no description).*
+:``flush_config_to`` (string):
+ *\- path to store config.*
+:``ignore_lock`` (boolean):
+ *\- if tank is locked ( *.lock file(s) presented in lock_dir), shoot nevertheless. Default:* ``False``
+:``lock_dir`` (string):
+ *\- directory to store *.lock files. Default:* ``/var/lock/``
+:``message`` (string):
+ *\- (no description).*
+:``operator`` (string):
+ *\- your username.*
+:``pid`` (integer):
+ *\- (no description).*
+:``taskset_path`` (string):
+ *\- (no description). Default:* ``taskset``
+:``uuid`` (string):
+ *\- (no description).*
+
+:allow_unknown:
+ False
+
+``version`` (string)
+--------------------
+*\- (no description).*
+
 Android
 =======
 
@@ -416,6 +462,10 @@ Influx
 ------------------------
 *\- (no description). Default:* ``500000``
 
+``custom_tags`` (dict)
+----------------------
+*\- (no description). Default:* ``{}``
+
 ``database`` (string)
 ---------------------
 *\- (no description). Default:* ``mydb``
@@ -428,6 +478,10 @@ Influx
 -------------------------
 *\- (no description). Default:* ``http://localhost/``
 
+``labeled`` (boolean)
+---------------------
+*\- (no description). Default:* ``False``
+
 ``password`` (string)
 ---------------------
 *\- (no description). Default:* ``root``
@@ -436,6 +490,10 @@ Influx
 ------------------
 *\- (no description). Default:* ``8086``
 
+``prefix_measurement`` (string)
+-------------------------------
+*\- (no description). Default:* ``""``
+
 ``tank_tag`` (string)
 ---------------------
 *\- (no description). Default:* ``unknown``
@@ -563,6 +621,8 @@ Phantom
 --------------------
 *\- Address of target. Format: [host]:port, [ipv4]:port, [ipv6]:port. Port is optional. Tank checks each test if port is available.* **Required.**
 
+:empty:
+ False
 :examples:
  ``127.0.0.1:8080``
 

docs/docs_gen.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 printf "================\nConfig reference\n================\n" > config_reference.rst
+tank-docs-gen -o config_reference.rst --title "Core" -a "../yandextank/core/config/schema.yaml"
 for p in `find ../yandextank/plugins/ \( -name "schema.py" -o -name "schema.yaml" \)`
 do
 tank-docs-gen -o config_reference.rst --title `echo $p | awk '{split($0,a,"\/"); print a[5]}'` -a $p

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='yandextank',
-    version='1.10.1',
+    version='1.10.2',
     description='a performance measurement tool',
     longer_description='''
 Yandex.Tank is a performance measurement and load testing automatization tool.
@@ -20,14 +20,12 @@
         'psutil>=1.2.1', 'requests>=2.5.1', 'paramiko>=1.16.0',
         'pandas>=0.18.0', 'numpy>=1.12.1', 'future>=0.16.0',
         'pip>=8.1.2',
-        'pyyaml>=3.12', 'cerberus==1.2', 'influxdb>=5.0.0',
-        'netort>=0.0.11',
+        'pyyaml>=3.12', 'cerberus==1.2', 'influxdb>=5.0.0', 'netort==0.2.4',
     ],
     setup_requires=[
-        'pytest-runner', 'flake8',
     ],
     tests_require=[
-        'pytest',
+        'pytest', 'pytest-runner', 'flake8',
     ],
     license='LGPLv2',
     classifiers=[

yandextank/api/apiworker.py

@@ -66,12 +66,13 @@ def configure(self, options):
         """ Make preparations before running Tank """
         self.options = options
         if self.options.get('lock_dir', None):
-            self.core.set_option(
-                self.core.SECTION, "lock_dir", self.options['lock_dir'])
+            self.core.set_option(self.core.SECTION, "lock_dir", self.options['lock_dir'])
+        if self.options.get('ignore_lock', None):
+            self.core.set_option(self.core.SECTION, 'ignore_lock', self.options['ignore_lock'])
 
         while True:
             try:
-                self.core.get_lock(self.options.get('ignore_lock', None))
+                self.core.get_lock()
                 break
             except Exception as exc:
                 if self.options.get('lock_fail', None):

yandextank/config_converter/converter.py

@@ -1,10 +1,11 @@
-from ConfigParser import ConfigParser
-import re
 import logging
+import re
+from ConfigParser import ConfigParser
+from functools import reduce
+
 import pkg_resources
 import yaml
 
-from functools import reduce
 from yandextank.common.util import recursive_dict_update
 from yandextank.validator.validator import load_plugin_schema, load_yaml_schema
 
@@ -489,10 +490,11 @@ def core_options(cfg_ini):
 
 def convert_ini(ini_file):
     cfg_ini = ConfigParser()
-    if isinstance(ini_file, str):
-        cfg_ini.read(ini_file)
-    else:
+    if isinstance(ini_file, file):
         cfg_ini.readfp(ini_file)
+    else:
+        cfg_ini.read(ini_file)
+
     ready_sections = enable_sections(combine_sections(parse_sections(cfg_ini)), core_options(cfg_ini))
 
     plugins_cfg_dict = {section.new_name: section.get_cfg_dict() for section in ready_sections}

yandextank/core/config/schema.yaml

@@ -25,9 +25,10 @@ core:
     flush_config_to:
       description: path to store config
       type: string
-    ignore_locks:
-      description: if tank is locked (*.lock file(s) presented in lock_dir), shoot nevertheless
+    ignore_lock:
+      description: if tank is locked ( *.lock file(s) presented in lock_dir), shoot nevertheless
       type: boolean
+      default: false
     uuid:
       type: string
     pid:

yandextank/core/consoleworker.py

@@ -282,15 +282,17 @@ class ConsoleTank:
     IGNORE_LOCKS = "ignore_locks"
 
     def __init__(self, options, ammofile):
-        overwrite_options = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {}
+        cli_kwargs = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {}
+        if options.ignore_lock:
+            cli_kwargs.setdefault('core', {})['ignore_lock'] = options.ignore_lock
         self.options = options
         self.baseconfigs_location = '/etc/yandex-tank'
         self.init_logging()
         self.log = logging.getLogger(__name__)
 
         if ammofile:
             self.log.debug("Ammofile: %s", ammofile)
-            overwrite_options['phantom'] = {
+            cli_kwargs['phantom'] = {
                 'use_caching': False,
                 'ammofile': ammofile
             }
@@ -299,7 +301,7 @@ def __init__(self, options, ammofile):
                                    options.option,
                                    options.no_rc,
                                    [],
-                                   overwrite_options,
+                                   cli_kwargs,
                                    options.patches)
 
         raw_cfg_file, raw_cfg_path = tempfile.mkstemp(suffix='_pre-validation-config.yaml')
@@ -387,7 +389,7 @@ def init_logging(self):
     def configure(self):
         while True:
             try:
-                self.core.get_lock(self.options.ignore_lock)
+                self.core.get_lock()
                 break
             except LockError:
                 if self.options.lock_fail:

yandextank/core/tankcore.py

@@ -209,7 +209,7 @@ def job(self):
                 gen = self.get_plugin_of_type(GeneratorPlugin)
             except KeyError:
                 logger.warning("Load generator not found")
-                gen = GeneratorPlugin(self, {}, None)
+                gen = GeneratorPlugin(self, {})
             # aggregator
             aggregator = TankAggregator(gen)
             self._job = Job(monitoring_plugins=monitorings,
@@ -292,7 +292,7 @@ def plugins_end_test(self, retcode):
         logger.info('Stopping monitoring')
         for plugin in self.job.monitoring_plugins:
             logger.info('Stopping %s', plugin)
-            retcode = plugin.end_test(retcode)
+            retcode = plugin.end_test(retcode) or retcode
             logger.info('RC after: %s', retcode)
 
         for plugin in [p for p in self.plugins.values() if
@@ -449,9 +449,9 @@ def get_lock_dir(self):
                 self.SECTION, "lock_dir")
         return os.path.expanduser(self.lock_dir)
 
-    def get_lock(self, force=False, lock_dir=None):
+    def get_lock(self, lock_dir=None):
         lock_dir = lock_dir if lock_dir else self.get_lock_dir()
-        if not force and self.is_locked(lock_dir):
+        if not self.get_option(self.SECTION, 'ignore_lock') and self.is_locked(lock_dir):
             raise LockError("Lock file(s) found")
 
         fh, self.lock_file = tempfile.mkstemp(
@@ -481,7 +481,9 @@ def is_locked(cls, lock_dir='/var/lock'):
                         running_cfg = yaml.load(f)
                     pid = running_cfg.get(TankCore.SECTION).get(cls.PID_OPTION)
                     if not pid:
-                        logger.warning('Failed to get {}.{} from lock file {}'.format(TankCore.SECTION))
+                        logger.warning('Failed to get {}.{} from lock file {}'.format(TankCore.SECTION,
+                                                                                      cls.PID_OPTION,
+                                                                                      full_name))
                     else:
                         if not pid_exists(int(pid)):
                             logger.debug(

yandextank/plugins/DataUploader/plugin.py

@@ -46,7 +46,6 @@ def identify_backend(cls, api_address):
             raise KeyError(
                 'Config section name doesn\'t match any of the patterns:\n%s' %
                 '\n'.join(['*%s*' % ptrn[0] for ptrn in clues]))
-        pass
 
 
 def chop(data_list, chunk_size):
@@ -223,7 +222,7 @@ def prepare_test(self):
             ammo_path = info.ammo_file
         else:
             ammo_path = os.path.realpath(info.ammo_file)
-        loop_count = info.loop_count
+        loop_count = int(info.loop_count)
 
         try:
             lp_job = self.lp_job
@@ -456,7 +455,7 @@ def __save_conf(self):
                 with open(config_filename) as config_file:
                     self.core.job.monitoring_plugin.set_option("config_contents", config_file.read())
         except AttributeError:  # pylint: disable=W0703
-            logger.info("Can't get monitoring config")
+            logger.info("Can't get monitoring config", exc_info=True)
 
         self.lp_job.send_config_snapshot(self.core.cfg_snapshot)
         self.core.config.save(
@@ -526,6 +525,7 @@ def __get_operator(self):
             raise
 
     def __get_api_client(self):
+        logging.info('Using {} backend'.format(self.backend_type))
         if self.backend_type == BackendTypes.LUNAPARK:
             client = APIClient
             self._api_token = None

yandextank/plugins/Influx/config/schema.yaml

@@ -30,4 +30,7 @@ labeled:
   type: boolean
 prefix_measurement:
   default: ""
-  type: string
+  type: string
+custom_tags:
+  default: {}
+  type: dict

yandextank/plugins/Influx/plugin.py

@@ -2,17 +2,17 @@
 # TODO: make the next two lines unnecessary
 # pylint: disable=line-too-long
 # pylint: disable=missing-docstring
+import datetime
 import logging
 import sys
-import datetime
-
 from uuid import uuid4
+
 from builtins import str
 from influxdb import InfluxDBClient
 
+from .decoder import Decoder
 from ...common.interfaces import AbstractPlugin, \
     MonitoringDataListener, AggregateResultListener
-from .decoder import Decoder
 
 logger = logging.getLogger(__name__)  # pylint: disable=C0103
 
@@ -30,7 +30,6 @@ def chop(data_list, chunk_size):
 
 class Plugin(AbstractPlugin, AggregateResultListener,
              MonitoringDataListener):
-
     SECTION = 'influx'
 
     def __init__(self, core, cfg):
@@ -47,6 +46,7 @@ def __init__(self, core, cfg):
         )
         self.labeled = self.get_option("labeled")
         self.prefix_measurement = self.get_option("prefix_measurement")
+        self.custom_tags = self.get_option("custom_tags")
         grafana_root = self.get_option("grafana_root")
         grafana_dashboard = self.get_option("grafana_dashboard")
         uuid = str(uuid4())
@@ -76,8 +76,16 @@ def on_aggregated_data(self, data, stats):
                 points = self.decoder.decode_aggregate_labeled(data, stats, self.prefix_measurement)
             else:
                 points = self.decoder.decode_aggregate(data, stats)
+            if len(self.custom_tags):
+                self.add_custom_tags(points)
             self.client.write_points(points, 's')
 
+    def add_custom_tags(self, points):
+        for p in points:
+            common_tags = p['tags']
+            common_tags.update(self.custom_tags)
+            p['tags'] = common_tags
+
     def monitoring_data(self, data_list):
         if self.client:
             if len(data_list) > 0:

yandextank/plugins/YASM/plugin.py

@@ -91,7 +91,7 @@ def __init__(self, alias, host, tags, signals=None, default_signals=True):
         custom_signals = signals if signals else []
         self.signals = DEFAULT_SIGNALS + custom_signals if default_signals else custom_signals
         self.host = host
-        self.tags = tags
+        self.tags = tags.strip(';')
         if len(self.signals) == 0:
             logger.warning('No signals specified for {} panel'.format(self.alias))
         self.as_dict = {self.host: {self.tags: self.signals}}