Merge pull request #605 from yandex/release

changes from Release

Dockerfile-TankBAT

@@ -0,0 +1,9 @@
+FROM ubuntu:16.04
+WORKDIR /yandextank
+RUN apt-get update && \
+    apt-get install -y python-pip
+RUN pip install --upgrade setuptools
+RUN pip install --upgrade pip
+RUN pip install pytest
+CMD pip install . && pytest -s
+# docker run -v /path/to/yandextank:/yandextank --name my_container my_image

docs/config_reference.rst

@@ -960,4 +960,12 @@ YASM
    :type:
     string
  :type:
-  dict
+  dict
+
+``timeout`` (integer)
+---------------------
+*\- (no description). Default:* ``120``
+
+``verbose_logging`` (boolean)
+-----------------------------
+*\- (no description). Default:* ``False``

yandextank/core/cli.py

@@ -82,6 +82,14 @@ def main():
         '--verbose',
         action='store_true',
         help="More console output, +debug messages")
+    parser.add_option(
+        '-p',
+        '--patch-cfg',
+        action='append',
+        help='Patch config with yaml snippet (similar to -o, but has full compatibility to\
+        and the exact scheme of yaml format config)',
+        dest='patches'
+    )
 
     completion_helper = CompletionHelperOptionParser()
     completion_helper.handle_request(parser)

yandextank/core/consoleworker.py

@@ -119,7 +119,7 @@ def parse_options(options):
         return [
             convert_single_option(key.strip(), value.strip())
             for key, value
-            in [option.split('=') for option in options]
+            in [option.split('=', 1) for option in options]
         ]
 
 
@@ -245,17 +245,24 @@ def patch_ini_config_with_monitoring(ini_config, mon_section_name):
         raise ex
 
 
-def load_tank_core(config_files, cmd_options, no_rc, depr_options, *other_opts):
+def load_tank_core(config_files, cmd_options, no_rc, depr_options, other_opts, patches=None):
+    if patches is None:
+        patches = []
     other_opts = list(other_opts) if other_opts else []
     config_files = config_files if len(config_files) > 0 else [DEFAULT_CONFIG]
     if no_rc:
-        configs = [load_cfg(cfg) for cfg in config_files] + other_opts + parse_options(cmd_options)
+        configs = [load_cfg(cfg) for cfg in config_files] +\
+            other_opts +\
+            parse_options(cmd_options) +\
+            [yaml.load(p) for p in patches]
     else:
         configs = [load_core_base_cfg()] +\
             load_local_base_cfgs() +\
-            [load_cfg(cfg) for cfg in config_files] + other_opts + parse_options(cmd_options)
-    return TankCore(configs,
-                    cfg_depr=get_depr_cfg(config_files, no_rc, cmd_options, depr_options))
+            [load_cfg(cfg) for cfg in config_files] +\
+            other_opts +\
+            parse_options(cmd_options) +\
+            [yaml.load(p) for p in patches]
+    return TankCore(configs)
 
 
 class ConsoleTank:
@@ -266,7 +273,6 @@ class ConsoleTank:
     def __init__(self, options, ammofile):
         overwrite_options = {'core': {'lock_dir': options.lock_dir}} if options.lock_dir else {}
         self.options = options
-        # self.lock_dir = options.lock_dir if options.lock_dir else '/var/lock'
         self.baseconfigs_location = '/etc/yandex-tank'
         self.init_logging()
         self.log = logging.getLogger(__name__)
@@ -282,7 +288,8 @@ def __init__(self, options, ammofile):
                                    options.option,
                                    options.no_rc,
                                    [],
-                                   overwrite_options)
+                                   overwrite_options,
+                                   options.patches)
 
         raw_cfg_file, raw_cfg_path = tempfile.mkstemp(suffix='_pre-validation-config.yaml')
         os.close(raw_cfg_file)

yandextank/core/tankcore.py

@@ -94,7 +94,7 @@ class TankCore(object):
     PID_OPTION = 'pid'
     UUID_OPTION = 'uuid'
 
-    def __init__(self, configs, artifacts_base_dir=None, artifacts_dir_name=None, cfg_depr=None):
+    def __init__(self, configs, artifacts_base_dir=None, artifacts_dir_name=None):
         """
 
         :param configs: list of dict
@@ -113,7 +113,6 @@ def __init__(self, configs, artifacts_base_dir=None, artifacts_dir_name=None, cf
         self.taskset_path = None
         self.taskset_affinity = None
         self._job = None
-        self.cfg_depr = cfg_depr
         self._cfg_snapshot = None
 
         self.interrupted = False

yandextank/plugins/YASM/config/schema.yaml

@@ -15,4 +15,10 @@ panels:
         type: list
       default_signals:
         type: boolean
-        default: true
+        default: true
+timeout:
+  type: integer
+  default: 120
+verbose_logging:
+  type: boolean
+  default: false

yandextank/plugins/YASM/plugin.py

@@ -38,7 +38,9 @@ def signals_stream(yasmapi_cfg):
     :return:
     '''
     for point in RtGolovanRequest(yasmapi_cfg.as_dict()):
-        # logger.info('YASM data:\n{}'.format(point.values))
+        logger.debug('received YASM data for {} for following hosts an tags: {}'.
+                     format(point.ts,
+                            {host: tags.keys() for host, tags in point.values.items()}))
         yield point.ts, {panel.alias: point.values[panel.host][panel.tags] for panel in yasmapi_cfg.panels}
 
 
@@ -107,7 +109,6 @@ def set_copy(self, key=None, value=None):
 
 
 class Plugin(MonitoringPlugin):
-    RECEIVE_TIMEOUT = 45
 
     def __init__(self, core, cfg, cfg_updater=None):
         super(Plugin, self).__init__(core, cfg)
@@ -116,6 +117,9 @@ def __init__(self, core, cfg, cfg_updater=None):
         self.stop_event = Event()
         self.last_ts = 0
         self.data_buffer = []
+        self.timeout = self.get_option('timeout')
+        if self.get_option('verbose_logging'):
+            logger.setLevel(logging.DEBUG)
 
     def add_listener(self, plugin):
         self.listeners.append(plugin)
@@ -165,7 +169,7 @@ def send_rest(self):
     def consumer(self):
         while not self.stop_event.is_set():
             try:
-                ts, data = self.data_queue.get(timeout=self.RECEIVE_TIMEOUT)
+                ts, data = self.data_queue.get(timeout=self.timeout)
                 # logger.info('Received monitoring data for {}'.format(ts))
                 self.last_ts = ts
                 self.data_buffer.append(monitoring_data(ts, data))