Reorder tests to prevent rediscovery of non-existing tests

test.sh

@@ -0,0 +1,18 @@
+fold_wrap() {
+    local header=$1; shift
+    local section=$1; shift
+    local cmd=$@
+    echo start $section $header
+    "$cmd"
+    local result=$?
+    echo end $section
+    return $result
+}
+
+example_command() {
+	set -e
+	echo Hallo
+	false
+	echo SHOULD NOT HAPPEN
+	set +e
+}

test_ir.py

@@ -1,6 +1,8 @@
 import unittest
 import jupyter_kernel_test as jkt
 
+from jupyter_client.manager import start_new_kernel
+
 class IRkernelTests(jkt.KernelTests):
     kernel_name = 'ir'
 
@@ -26,16 +28,6 @@ class InstallspecTests(jkt.KernelTests):
 
     code_hello_world = 'print("hello, world")'
 
-class DisplaySystemTests(jkt.KernelTests):
-    kernel_name = 'ir'
-
-    language_name = 'R'
-
-    code_display_data = [
-    {'code': 'options(jupyter.rich_display = FALSE);cat("a")', 'mime': {'text/plain':'a'}},
-    {'code': '"a"', 'mime': {'text/plain':'"a"'}},
-    {'code': '1:3', 'mime': {'text/plain':'[1] 1 2 3'}},
-    ]
 
 class AbstractIRKernel(jkt.KernelTests):
     kernel_name = 'ir'
@@ -53,7 +45,15 @@ def _execute_code(self, code):
         self.assertEqual(output_msgs[0]['msg_type'], 'display_data')
         return reply, output_msgs
 
-class PlotExistsTests(AbstractIRKernel):
+
+class IndependendTests(AbstractIRKernel):
+
+    """This contains tests cases which do not alter the kernel environment.
+
+    They are all in one class so that unittest test case discovery does not
+    discover and skip over the default cases all the time...
+    """
+
     def test_irkernel_plots(self):
         code = "plot(1:3)"
         reply, output_msgs = self._execute_code(code)
@@ -67,19 +67,6 @@ def test_irkernel_plots(self):
         self.assertEqual(len(output_msgs[0]['content']['metadata']['image/svg+xml']), 1)
         self.assertEqual(output_msgs[0]['content']['metadata']['image/svg+xml']['isolated'], True)
 
-class OnlyPNGPlotsTests(AbstractIRKernel):
-
-    def test_irkernel_plots(self):
-        code = """
-options(jupyter.plot_mimetypes = 'image/png')
-plot(1:3)
-"""
-        reply, output_msgs = self._execute_code(code)
-        # Only png
-        self.assertEqual(len(output_msgs[0]['content']['data']), 1)
-        self.assertIn('image/png', output_msgs[0]['content']['data'])
-
-class DefaultRichOutput(AbstractIRKernel):
 
     def test_irkernel_default_rich_output(self):
         code = """
@@ -89,7 +76,43 @@ def test_irkernel_default_rich_output(self):
         # we currently send three formats: text/plain, html, and latex
         self.assertEqual(len(output_msgs[0]['content']['data']), 3)
 
-class NoRichOutput(AbstractIRKernel):
+
+class OptionsDependendTests(AbstractIRKernel):
+    """Test cases which need to get a new kernel because the options are changed"""
+
+    def setUp(self):
+        self.km, self.kc = start_new_kernel(kernel_name=self.kernel_name)
+
+    def tearDown(self):
+        self.kc.stop_channels()
+        self.km.shutdown_kernel()
+
+    # overwrite defaults to prevent errors on already shutdown kernels
+    @classmethod
+    def setUpClass(cls):
+        pass
+
+    @classmethod
+    def tearDownClass(cls):
+        pass
+
+
+    code_display_data = [
+    {'code': 'options(jupyter.rich_display = FALSE);cat("a")', 'mime': {'text/plain':'a'}},
+    {'code': '"a"', 'mime': {'text/plain':'"a"'}},
+    {'code': '1:3', 'mime': {'text/plain':'[1] 1 2 3'}},
+    ]
+
+    def test_irkernel_only_PNG_plots(self):
+        code = """
+options(jupyter.plot_mimetypes = 'image/png')
+plot(1:3)
+"""
+        reply, output_msgs = self._execute_code(code)
+        # Only png
+        self.assertEqual(len(output_msgs[0]['content']['data']), 1)
+        self.assertIn('image/png', output_msgs[0]['content']['data'])
+
 
     def test_irkernel_no_rich_output(self):
         code = """
@@ -100,5 +123,6 @@ def test_irkernel_no_rich_output(self):
         # only text/plain
         self.assertEqual(len(output_msgs[0]['content']['data']), 1)
 
+
 if __name__ == '__main__':
     unittest.main()