Revamped test runner and made log time-stamps relative.

The preferred way to run the test suite is through `test.py` in the
`deploy` folder. That script now also supports running individual
test scripts (a.k.a. test "groups"), such as `test_node.py` by passing
"node" as a command-line argument. Passing "--log" activates output of
log messages (with debug-level granularity).

The `import parent` statement in the test scripts was removed, as well
as the corresponding helper module, which used to add the `mph` folder
to `sys.path`. This means running test scripts directly via, for
example, `python tests/test_node.py` from the root folder, will no
longer work out of the box as the script will then fail to `import mph`
(unless MPh has been installed separately via Pip or its folder been
added manually to the PYTHONPATH environment variable).

The setup of log output to the console during test runs was refactored.
The test scripts now only accept one command-line option, `--log` just
like the test runner, instead of `log` and `debug` before. Time-stamps
are now relative with respect to the start of an individual test script,
instead of the absolute wall-clock time they were previously.

deploy/coverage.py

@@ -11,7 +11,7 @@
 If the coverage report file (`coverage.xml`) already exists, we instead
 render it as HTML for inspection. This is helpful during development.
 The coverage report may also be uploaded to an online service such as
-CodeCov for each published release or even commit.
+CodeCov for each published release or even commit, see `codecov.py`.
 """
 
 from subprocess import run

deploy/test.py

@@ -1,33 +1,93 @@
 """
 Runs all tests in the intended order.
 
-This script does not run the tests via pyTest, but just executes the
-test scripts as is, in a subprocess. (pyTest is however needed for some
-test fixtures, so it must be installed.) We impose the intended order,
-from the most basic functionality to the high-level abstractions.
-
-Pass 'log' as a command-line argument to have the scripts print log
-messages to the console while the tests are running. Pass 'debug' to
-increase the verbosity of the log.
+Each test script (in the `tests` folder) contains a group of tests.
+These scripts must be run in separate processes as most of them start
+and stop the Java virtual machine, which can only be done once per
+process. This is why simply calling pyTest (with `python -m pytest`
+in the root folder) will not work.
+
+This script here runs each test group in a new subprocess. It also
+imposes a logical order: from the tests covering the most most basic
+functionality to the high-level abstractions.
+
+Here, as opposed to the similar script `coverage.py`, we don't actually
+run the tests through pyTest. Rather, we run the scripts directly so
+that the output is less verbose. Note, however, that pyTest still needs
+to be installed as some of the test fixtures require it.
+
+The verbosity can be increased by passing `--log` as a command-line
+argument. This will display the log messages produced by MPh as the
+tests are running. You can also pass the name of a test group to run
+only that one. For example, passing "model" will only run the tests
+defined in `test_model.py`.
 """
 
 from subprocess import run
 from pathlib    import Path
 from timeit     import default_timer as now
-import sys
+from argparse   import ArgumentParser
+from sys        import executable as python
+from sys        import exit
+from os         import environ, pathsep
+
+
+# Define order of test groups.
+groups = ['meta', 'config', 'discovery', 'server', 'session', 'standalone',
+          'client', 'multi', 'node', 'model', 'exit']
+
+# Determine path of project root folder.
+here = Path(__file__).resolve().parent
+root = here.parent
+
+# Run MPh in project folder, not a possibly different installed version.
+if 'PYTHONPATH' in environ:
+    environ['PYTHONPATH'] = str(root) + pathsep + environ['PYTHONPATH']
+else:
+    environ['PYTHONPATH'] = str(root)
 
-tests = ['meta', 'config', 'discovery', 'server', 'session', 'standalone',
-         'client', 'multi', 'node', 'model', 'exit']
+# Parse command-line arguments.
+parser = ArgumentParser(prog='test.py',
+                        description='Runs the MPh test suite.',
+                        add_help=False,
+                        allow_abbrev=False)
+parser.add_argument('--help',
+                    help='Show this help message.',
+                    action='help')
+parser.add_argument('--log',
+                    help='Display log output.',
+                    action='store_true')
+parser.add_argument('--groups',
+                    help='List all test groups.',
+                    action='store_true')
+parser.add_argument('group',
+                    help='Run only this group of tests.',
+                    nargs='?')
+arguments = parser.parse_args()
+if arguments.groups:
+    for group in groups:
+        print(group)
+    exit()
+if arguments.group:
+    group = arguments.group
+    if group.startswith('test_'):
+        group = group[5:]
+    if group.endswith('.py'):
+        group = group[:-3]
+    groups = [group]
+options = []
+if arguments.log:
+    options.append('--log')
 
-folder = Path(__file__).parent.parent / 'tests'
-python = sys.executable
-arguments = sys.argv[1:]
-for test in tests:
-    print(f'test_{test}')
+# Run each test group in new process.
+for group in groups:
+    if groups.index(group) > 0:
+        print()
+    print(f'Running test group "{group}".')
     t0 = now()
-    process = run([python, f'test_{test}.py'] + arguments, cwd=folder)
+    process = run([python, f'test_{group}.py'] + options, cwd=root/'tests')
     if process.returncode == 0:
         print(f'Passed in {now()-t0:.0f} s.')
     else:
         print(f'Failed after {now()-t0:.0f} s.')
-    print()
+        exit(1)

tests/exit_client_exc.py

@@ -1,6 +1,5 @@
 """Process exiting with exception after starting the client."""
 
-import parent # noqa F401
 import mph
 
 mph.start()

tests/exit_client_sys.py

@@ -1,6 +1,5 @@
 """Process exiting via `sys.exit()` after starting the client."""
 
-import parent # noqa F401
 import mph
 import sys
 

tests/exit_nojvm_exc.py

@@ -1,6 +1,5 @@
 """Process exiting with exception when no Java VM is running."""
 
-import parent # noqa F401
-import mph    # noqa F401
+import mph # noqa F401
 
 raise RuntimeError

tests/exit_nojvm_sys.py

@@ -1,7 +1,6 @@
 """Process exiting via `sys.exit()` with no Java VM running."""
 
-import parent # noqa F401
-import mph    # noqa F401
+import mph # noqa F401
 import sys
 
 sys.exit(2)

tests/fixtures.py

@@ -2,11 +2,12 @@
 
 import logging
 import warnings
-from io import StringIO
+import io
 import sys
 
 
 class logging_disabled:
+    """Suppresses log messages issued in this context."""
 
     def __enter__(self):
         self.level = logging.getLogger().level
@@ -18,6 +19,7 @@ def __exit__(self, type, value, traceback):
 
 
 class warnings_disabled:
+    """Suppresses warnings raised in this context."""
 
     def __enter__(self):
         warnings.simplefilter('ignore')
@@ -28,10 +30,11 @@ def __exit__(self, type, value, traceback):
 
 
 class capture_stdout:
+    """Captures text written to `sys.stdout` in this context."""
 
     def __enter__(self):
         self.stdout = sys.stdout
-        self.buffer = StringIO()
+        self.buffer = io.StringIO()
         sys.stdout = self.buffer
         return self
 
@@ -40,3 +43,24 @@ def __exit__(self, type, value, traceback):
 
     def text(self):
         return self.buffer.getvalue()
+
+
+original_records = logging.getLogRecordFactory()
+
+
+def timed_records(*args, **kwargs):
+    """Adds a (relative) `timestamp` to the log record attributes."""
+    record = original_records(*args, **kwargs)
+    (minutes, seconds) = divmod(record.relativeCreated/1000, 60)
+    record.timestamp = f'{minutes:02.0f}:{seconds:06.3f}'
+    return record
+
+
+def setup_logging():
+    """Sets up logging to console if `--log` command-line argument present."""
+    if '--log' not in sys.argv[1:]:
+        return
+    logging.setLogRecordFactory(timed_records)
+    logging.basicConfig(
+        level   = logging.DEBUG,
+        format  = '[%(timestamp)s] %(message)s')

tests/test_client.py

@@ -3,13 +3,11 @@
 ########################################
 # Dependencies                         #
 ########################################
-import parent # noqa F401
 import mph
 from fixtures import logging_disabled
+from fixtures import setup_logging
 from pytest import raises
 from pathlib import Path
-from sys import argv
-import logging
 
 
 ########################################
@@ -182,14 +180,7 @@ def test_connect():
 ########################################
 
 if __name__ == '__main__':
-
-    arguments = argv[1:]
-    if 'log' in arguments:
-        logging.basicConfig(
-            level   = logging.DEBUG if 'debug' in arguments else logging.INFO,
-            format  = '[%(asctime)s.%(msecs)03d] %(message)s',
-            datefmt = '%H:%M:%S')
-
+    setup_logging()
     test_init()
     test_load()
     test_create()

tests/test_config.py

@@ -3,13 +3,11 @@
 ########################################
 # Dependencies                         #
 ########################################
-import parent # noqa F401
 import mph
 from fixtures import logging_disabled
+from fixtures import setup_logging
 from pytest import raises
 from pathlib import Path
-from sys import argv
-import logging
 
 
 ########################################
@@ -65,14 +63,7 @@ def test_load():
 ########################################
 
 if __name__ == '__main__':
-
-    arguments = argv[1:]
-    if 'log' in arguments:
-        logging.basicConfig(
-            level   = logging.DEBUG if 'debug' in arguments else logging.INFO,
-            format  = '[%(asctime)s.%(msecs)03d] %(message)s',
-            datefmt = '%H:%M:%S')
-
+    setup_logging()
     test_option()
     test_location()
     test_save()

tests/test_discovery.py

@@ -3,10 +3,8 @@
 ########################################
 # Dependencies                         #
 ########################################
-import parent # noqa F401
 import mph
-import logging
-from sys import argv
+from fixtures import setup_logging
 
 
 ########################################
@@ -33,13 +31,6 @@ def test_backend():
 ########################################
 
 if __name__ == '__main__':
-
-    arguments = argv[1:]
-    if 'log' in arguments:
-        logging.basicConfig(
-            level   = logging.DEBUG if 'debug' in arguments else logging.INFO,
-            format  = '[%(asctime)s.%(msecs)03d] %(message)s',
-            datefmt = '%H:%M:%S')
-
+    setup_logging()
     test_parse()
     test_backend()

tests/test_exit.py

@@ -3,12 +3,10 @@
 ########################################
 # Dependencies                         #
 ########################################
-import parent # noqa F401
+from fixtures import setup_logging
 from subprocess import run, PIPE
 from pathlib import Path
-from sys import argv
 from sys import executable as python
-import logging
 
 
 ########################################
@@ -61,14 +59,7 @@ def test_exit_client_exc():
 ########################################
 
 if __name__ == '__main__':
-
-    arguments = argv[1:]
-    if 'log' in arguments:
-        logging.basicConfig(
-            level   = logging.DEBUG if 'debug' in arguments else logging.INFO,
-            format  = '[%(asctime)s.%(msecs)03d] %(message)s',
-            datefmt = '%H:%M:%S')
-
+    setup_logging()
     test_exit_nojvm_sys()
     test_exit_nojvm_exc()
     test_exit_client_sys()

tests/test_meta.py

@@ -1,10 +1,17 @@
 """Tests the `meta` module."""
 
-import parent # noqa F401
+########################################
+# Dependencies                         #
+########################################
 from mph import meta
+from fixtures import setup_logging
 import re
 
 
+########################################
+# Tests                                #
+########################################
+
 def test_meta():
     fields = ['title', 'synopsis', 'version', 'author', 'copyright', 'license']
     for field in fields:
@@ -16,3 +23,12 @@ def test_meta():
     assert meta.author
     assert meta.copyright
     assert meta.license
+
+
+########################################
+# Main                                 #
+########################################
+
+if __name__ == '__main__':
+    setup_logging()
+    test_meta()