gh-5 multiengine tests feature

lib/colorer.py

@@ -47,6 +47,7 @@ class SchemaAscetic(CSchema):
         'test_skip': {'fgcolor': 'grey'},
         'test_disa': {'fgcolor': 'grey'},
         'error':     {'fgcolor': 'red'},
+        'test_var':  {'fgcolor': 'yellow'},
     }
 
 class SchemaPretty(CSchema):
@@ -70,7 +71,8 @@ class SchemaPretty(CSchema):
         'serv_text': {'fgcolor': 'lmagenta'},
         'version':   {'fgcolor': 'yellow', 'bold':True},
         'tr_text':   {'fgcolor': 'green'},
-        'log':       {'fgcolor': 'grey'}
+        'log':       {'fgcolor': 'grey'},
+        'test_var':  {'fgcolor': 'yellow'},
     }
 
 class Colorer(object):

lib/preprocessor.py

@@ -21,14 +21,15 @@ def __str__(self):
         return "lua preprocessor error: " + repr(self.value)
 
 class TestState(object):
-    def __init__(self, suite_ini, default_server, create_server):
+    def __init__(self, suite_ini, default_server, create_server, params = {}):
         self.delimiter = ''
         self.suite_ini = suite_ini
         self.environ = Namespace()
         self.operation = False
         self.create_server = create_server
         self.servers = { 'default': default_server }
         self.connections = {}
+        self.run_params = params
         if default_server is not None:
             self.connections =  { 'default': default_server.admin }
             # curcon is an array since we may have many connections
@@ -66,6 +67,9 @@ def parse_preprocessor(self, string):
         elif token == 'switch':
             server = lexer.get_token()
             return self.switch(server)
+        elif token == 'config':
+            var_name = lexer.get_token()
+            return self.run_params
         token_store.append(token)
         token = lexer.get_token()
         if token == 'server':

lib/tarantool_server.py

@@ -82,7 +82,10 @@ def send_command(command):
         ts.cleanup()
 
     def execute(self, server):
-        ts = TestState(self.suite_ini, server, TarantoolServer)
+        ts = TestState(
+            self.suite_ini, server, TarantoolServer,
+            self.run_params
+        )
         self.inspector.set_parser(ts)
         lua = gevent.Greenlet.spawn(self.exec_loop, ts)
         lua.join()
@@ -616,10 +619,22 @@ def test_debug(self):
 
     def find_tests(self, test_suite, suite_path):
         test_suite.ini['suite'] = suite_path
-        tests  = [PythonTest(k, test_suite.args, test_suite.ini) \
-                for k in sorted(glob.glob(os.path.join(suite_path, "*.test.py" )))]
-        tests += [LuaTest(k, test_suite.args, test_suite.ini)    \
-                for k in sorted(glob.glob(os.path.join(suite_path, "*.test.lua")))]
+        get_tests = lambda x: sorted(glob.glob(os.path.join(suite_path, x)))
+        tests  = [PythonTest(k, test_suite.args, test_suite.ini)
+            for k in get_tests("*.test.py")
+        ]
+        for k in get_tests("*.test.lua"):
+            runs = test_suite.get_multirun_params(k)
+            is_correct = lambda x: test_suite.args.conf is None or \
+                test_suite.args.conf == x
+            if runs:
+                tests.extend([LuaTest(
+                    k, test_suite.args,
+                    test_suite.ini, runs[r], r
+                ) for r in runs.keys() if is_correct(r)])
+            else:
+                tests.append(LuaTest(k, test_suite.args, test_suite.ini))
+
         test_suite.tests = []
         # don't sort, command line arguments must be run in
         # the specified order

lib/test.py

@@ -69,7 +69,7 @@ class Test:
     """
     rg = re.compile('\.test.*')
 
-    def __init__(self, name, args, suite_ini):
+    def __init__(self, name, args, suite_ini, params={}, conf_name=None):
         """Initialize test properties: path to test file, path to
         temporary result file, path to the client program, test status."""
         self.name = name
@@ -87,6 +87,8 @@ def __init__(self, name, args, suite_ini):
         self.is_equal_result = None
         self.is_valgrind_clean = True
         self.is_terminated = False
+        self.run_params = params
+        self.conf_name = conf_name
 
     def passed(self):
         """Return true if this test was run successfully."""

lib/test_suite.py

@@ -2,6 +2,7 @@
 import re
 import sys
 import time
+import json
 import shutil
 import difflib
 import threading
@@ -33,6 +34,30 @@ class TestSuite:
     server for this suite, the client program to execute individual
     tests and other suite properties. The server is started once per
     suite."""
+    def get_multirun_conf(self, suite_path):
+        conf_name = self.ini.get('config', None)
+        if conf_name is None:
+            return None
+
+        path = os.path.join(suite_path, conf_name)
+        result = None
+        with open(path) as cfg:
+            try:
+                result = json.load(cfg)
+            except ValueError:
+                raise RuntimeError('Ivalid multirun json')
+        return result
+
+    def get_multirun_params(self, test_path):
+        test = test_path.split('/')[-1]
+        if self.multi_run is None:
+            return
+        result = self.multi_run.get(test, None)
+        if result is not None:
+            return result
+        result = self.multi_run.get('*', None)
+        return result
+
 
     def __init__(self, suite_path, args):
         """Initialize a test suite: check that it exists and contains
@@ -52,6 +77,8 @@ def __init__(self, suite_path, args):
         config.read(os.path.join(suite_path, "suite.ini"))
         self.ini.update(dict(config.items("default")))
         self.ini.update(self.args.__dict__)
+        self.multi_run = self.get_multirun_conf(suite_path)
+
         if self.args.stress is None and self.ini['core'] == 'stress':
             return
 
@@ -88,10 +115,11 @@ def run_all(self):
         self.server.cls = self.tests[0].__class__
         self.server.deploy(silent=False)
 
-        longsep = '='*70
-        shortsep = '-'*60
+        longsep = '='*80
+        shortsep = '-'*75
         color_stdout(longsep, "\n", schema='separator')
         color_stdout("TEST".ljust(48), schema='t_name')
+        color_stdout("PARAMS\t\t", schema='test_var')
         color_stdout("RESULT\n", schema='test_pass')
         color_stdout(shortsep, "\n", schema='separator')
         failed_tests = []
@@ -110,6 +138,10 @@ def run_all(self):
                 )
                 # for better diagnostics in case of a long-running test
 
+                conf = 'none'
+                if test.run_params:
+                    conf = test.conf_name
+                color_stdout("%s" % conf.ljust(16), schema='test_var')
                 test_name = os.path.basename(test.name)
                 if (test_name in self.ini["disabled"]
                     or not self.server.debug and test_name in self.ini["release_disabled"]

test-run.py

@@ -127,6 +127,13 @@ def __init__(self):
                action='store_true',
                help="""Enable long run tests""")
 
+        parser.add_argument(
+                "--conf",
+                dest="conf",
+                default=None,
+                help="""Force set test configuration mode"""
+        )
+
         self.args = parser.parse_args()
         self.check()
 

test_run.lua

@@ -54,6 +54,15 @@ local function switch(self, node)
     return self:cmd('set connection ' .. node)
 end
 
+local function get_cfg(self, name)
+    if self.run_conf == nil then
+        self.run_conf = json.decode(
+            self:cmd('config ' .. name)
+        )
+    end
+    return self.run_conf[name]
+end
+
 local function new(host, port)
     local inspector = {}
 
@@ -76,6 +85,7 @@ local function new(host, port)
     inspector.get_lsn = get_lsn
     inspector.wait_lsn = wait_lsn
     inspector.switch = switch
+    inspector.get_cfg = get_cfg
     return inspector
 end