Improve tests

.travis.yml

@@ -5,15 +5,23 @@ matrix:
         - os: linux
         - os: osx
     - language: python
-      python:
-        - "3.6"
-        - "3.7"
-        - "3.8"
-      # command to install dependencies
+      python: "3.6"
+      install:
+        - pip install --upgrade pip
+        - pip3 install -e .
+      script: pytest
+    - language: python
+      python: "3.7"
       install:
         - pip install --upgrade pip
         - pip3 install -e .
-      # command to run tests
       script:
         - pytest
-        - TEMCI_TEST_CMD=1 pytest
+        - TEMCI_TEST_CMD=1 pytest
+    - language: python
+      python: "3.8"
+      install:
+        - pip install --upgrade pip
+        - pip3 install -e .
+      script:
+        - pytest

temci/report/stats.py

@@ -498,7 +498,10 @@ def _store_as_image(self, filename: str, fig_width: float, fig_height: float) ->
         if not filename.endswith(self.img_filename_ending):
             filename += self.img_filename_ending
         self.reset_plt()
-        plt.savefig(filename)
+        try:
+            plt.savefig(filename)
+        except FileNotFoundError:
+            print("hi")
         self.reset_plt()
         return os.path.realpath(filename)
 

tests/test_report.py

@@ -3,7 +3,8 @@
 """
 import json
 
-from tests.utils import run_temci
+from tests.utils import run_temci, run_temci_proc
+
 
 def test_console_reporter_auto_mode():
     d = lambda d: {
@@ -43,6 +44,20 @@ def test_html2_with_single():
     }).file_contents
 
 
+def test_all_reporters():
+    from temci.report.report import ReporterRegistry
+    for name, rep in ReporterRegistry.registry.items():
+        print(name)
+        run_temci_proc("report --reporter {} in.yaml".format(name), files={
+            "in.yaml": [
+                {
+                    "attributes": {"description": "XYZ"},
+                    "data": {"p": [1, 2]}
+                }
+            ]
+        })
+
+
 def test_codespeed_reporter():
     d = lambda: {
         "attributes": {"description": "XYZ"},

tests/test_run.py

@@ -38,15 +38,14 @@ def test_errorneous_run():
 
 
 def test_check_tag_attribute():
-    with pytest.raises(TypeError):
-        assert run_temci("exec bla.yaml --runs 1", files={
-            "bla.yaml": [
-                {
-                    "run_config": {"cmd": "echo 1"},
-                    "attributes": {"tags": "slow"}
-                }
-            ]
-        }).ret_code != 0
+    assert run_temci("exec bla.yaml --runs 1", files={
+        "bla.yaml": [
+            {
+                "run_config": {"cmd": "echo 1"},
+                "attributes": {"tags": "slow"}
+            }
+        ]
+    }, expect_success=False).ret_code != 0
 
 
 def test_included_blocks():

tests/utils.py

@@ -4,6 +4,7 @@
 import sys
 
 import tempfile
+import traceback
 from typing import Dict, Union, NamedTuple, Tuple, Any
 
 import yaml
@@ -17,7 +18,7 @@
 from temci.utils.settings import Settings
 
 
-from temci.scripts.cli import cli
+from temci.scripts.cli import cli, ErrorCode
 
 
 class Result(NamedTuple):
@@ -42,7 +43,7 @@ def run_temci_proc(args: str, settings: dict = None, files: Dict[str, Union[dict
     """
     with tempfile.TemporaryDirectory() as d:
         _store_files(files, str(d))
-        cmd = "temci " + args
+        cmd = "python3 {}/temci/scripts/cli.py {}".format(os.path.dirname(os.path.dirname(__file__)), args)
         if settings is not None:
             with open(d + "/settings.yaml", "w") as f:
                 yaml.dump(settings, f)
@@ -92,7 +93,7 @@ def _load_files(files: Dict[str, Any], d: str = ".") -> Tuple[Dict[str, str], Di
 
 
 def run_temci_click(args: str, settings: dict = None, files: Dict[str, Union[dict, list, str]] = None,
-                    expect_success: bool = True, misc_env: Dict[str, str] = None) \
+                    expect_success: bool = True, misc_env: Dict[str, str] = None, raise_exc: bool = False) \
         -> Result:
     """
     Run temci with the passed arguments
@@ -119,21 +120,32 @@ def run_temci_click(args: str, settings: dict = None, files: Dict[str, Union[dic
         env.update(misc_env or {})
         args = sys.argv.copy()
         sys.argv = shlex.split("temci " + cmd)
-        result = runner.invoke(cli, cmd.replace(" ", " --settings settings.yaml ", 1), env=env, catch_exceptions=True)
+        err_code = None
+        exc = None
+        try:
+            result = runner.invoke(cli, cmd.replace(" ", " --settings settings.yaml ", 1), env=env, catch_exceptions=True)
+        except Exception as ex:
+            print("".join(traceback.format_exception(None, ex, ex.__traceback__)), sys.stderr)
+            err_code = ErrorCode.TEMCI_ERROR
+            exc = ex
         sys.argv = args
         file_contents, yaml_contents = _load_files(files)
-        ret = Result(result.output.strip(), str(result.stderr_bytes).strip(), result.exit_code, file_contents, yaml_contents)
+        ret = Result(result.output.strip(), str(result.stderr_bytes).strip(),
+                     err_code if err_code is not None else result.exit_code, file_contents, yaml_contents)
         Settings().load_from_dict(prior)
         if result.exception and not isinstance(result.exception, SystemExit):
             print(repr(ret))
-            raise result.exception
+            if raise_exc:
+                raise result.exception
+        if exc and raise_exc:
+            raise exc
         if expect_success:
             assert result.exit_code == 0, repr(ret)
         return ret
 
 
 def run_temci(args: str, settings: dict = None, files: Dict[str, Union[dict, list, str]] = None,
-              expect_success: bool = True, misc_env: Dict[str, str] = None) \
+              expect_success: bool = True, misc_env: Dict[str, str] = None, raise_exc: bool = False) \
         -> Result:
     """
     Run temci with the passed arguments
@@ -147,5 +159,5 @@ def run_temci(args: str, settings: dict = None, files: Dict[str, Union[dict, lis
     """
     if os.getenv("TEMCI_TEST_CMD", "0") == "1":
         return run_temci_proc(args, settings, files, expect_success, misc_env=misc_env)
-    return run_temci_click(args, settings, files, expect_success, misc_env=misc_env)
+    return run_temci_click(args, settings, files, expect_success, misc_env=misc_env, raise_exc=raise_exc)