Fail in remote if during a shutdown there are non-daemon threads present

Also add tracing option to trace remote tests
Add integration tests to cover

fixes #822

src/integrationtest/python/issue_822_tests.py

@@ -0,0 +1,76 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of PyBuilder
+#
+#   Copyright 2011-2021 PyBuilder Team
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import textwrap
+import unittest
+
+from itest_support import IntegrationTestSupport
+from pybuilder.errors import BuildFailedException
+
+
+class Issue822Test(IntegrationTestSupport):
+    def test(self):
+        self.write_build_file("""
+from pybuilder.core import use_plugin, init
+
+use_plugin("python.core")
+use_plugin("python.unittest")
+
+@init
+def init (project):
+    project.set_property("verbose", True)
+    project.set_property("remote_debug", 2)
+    project.set_property("remote_tracing", 1)
+        """)
+
+        self.create_directory("src/main/python")
+        self.create_directory("src/unittest/python")
+        self.write_file("src/main/python/code.py", textwrap.dedent(
+            """
+            import threading
+            import time
+            class TestThread(threading.Thread):
+                def __init__(self):
+                    super().__init__()
+
+                def run(self):
+                    time.sleep(7)
+
+            def run_code():
+                TestThread().start()
+
+            """))
+        self.write_file("src/unittest/python/code_tests.py", textwrap.dedent(
+            """
+            import unittest
+            import code
+
+            class CodeTests(unittest.TestCase):
+                def test_code(self):
+                    code.run_code()
+            """
+        ))
+        reactor = self.prepare_reactor()
+        with self.assertRaises(BuildFailedException) as raised_ex:
+            reactor.build("verify")
+
+        self.assertEqual(raised_ex.exception.message, "Unittest tool failed with exit code 1")
+
+
+if __name__ == "__main__":
+    unittest.main()

src/integrationtest/python/issue_822_with_remote_debug_tests.py

@@ -0,0 +1,75 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of PyBuilder
+#
+#   Copyright 2011-2021 PyBuilder Team
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import textwrap
+import unittest
+
+from itest_support import IntegrationTestSupport
+from pybuilder.errors import BuildFailedException
+
+
+class Issue822Test(IntegrationTestSupport):
+    def test(self):
+        self.write_build_file("""
+from pybuilder.core import use_plugin, init
+
+use_plugin("python.core")
+use_plugin("python.unittest")
+
+@init
+def init (project):
+    project.set_property("verbose", True)
+    project.set_property("remote_debug", 1)
+        """)
+
+        self.create_directory("src/main/python")
+        self.create_directory("src/unittest/python")
+        self.write_file("src/main/python/code.py", textwrap.dedent(
+            """
+            import threading
+            import time
+            class TestThread(threading.Thread):
+                def __init__(self):
+                    super().__init__()
+
+                def run(self):
+                    time.sleep(7)
+
+            def run_code():
+                TestThread().start()
+
+            """))
+        self.write_file("src/unittest/python/code_tests.py", textwrap.dedent(
+            """
+            import unittest
+            import code
+
+            class CodeTests(unittest.TestCase):
+                def test_code(self):
+                    code.run_code()
+            """
+        ))
+        reactor = self.prepare_reactor()
+        with self.assertRaises(BuildFailedException) as raised_ex:
+            reactor.build("verify")
+
+        self.assertEqual(raised_ex.exception.message, "Unittest tool failed with exit code 1")
+
+
+if __name__ == "__main__":
+    unittest.main()

src/integrationtest/python/issue_822_with_tracing_tests.py

@@ -0,0 +1,75 @@
+#   -*- coding: utf-8 -*-
+#
+#   This file is part of PyBuilder
+#
+#   Copyright 2011-2021 PyBuilder Team
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import textwrap
+import unittest
+
+from itest_support import IntegrationTestSupport
+from pybuilder.errors import BuildFailedException
+
+
+class Issue822Test(IntegrationTestSupport):
+    def test(self):
+        self.write_build_file("""
+from pybuilder.core import use_plugin, init
+
+use_plugin("python.core")
+use_plugin("python.unittest")
+
+@init
+def init (project):
+    project.set_property("verbose", True)
+    project.set_property("remote_tracing", 1)
+        """)
+
+        self.create_directory("src/main/python")
+        self.create_directory("src/unittest/python")
+        self.write_file("src/main/python/code.py", textwrap.dedent(
+            """
+            import threading
+            import time
+            class TestThread(threading.Thread):
+                def __init__(self):
+                    super().__init__()
+
+                def run(self):
+                    time.sleep(7)
+
+            def run_code():
+                TestThread().start()
+
+            """))
+        self.write_file("src/unittest/python/code_tests.py", textwrap.dedent(
+            """
+            import unittest
+            import code
+
+            class CodeTests(unittest.TestCase):
+                def test_code(self):
+                    code.run_code()
+            """
+        ))
+        reactor = self.prepare_reactor()
+        with self.assertRaises(BuildFailedException) as raised_ex:
+            reactor.build("verify")
+
+        self.assertEqual(raised_ex.exception.message, "Unittest tool failed with exit code 1")
+
+
+if __name__ == "__main__":
+    unittest.main()

src/main/python/pybuilder/plugins/core_plugin.py

@@ -31,6 +31,7 @@ def init(project):
     project.set_property("dir_logs", jp("$dir_target", "logs"))
 
     project.set_property_if_unset("remote_debug", 0)
+    project.set_property_if_unset("remote_tracing", 0)
 
     project.write_report = partial(write_report, project)
 

src/main/python/pybuilder/plugins/python/remote_tools/__init__.py

@@ -16,7 +16,6 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-
 from pybuilder.remote import Process, PipeShutdownError, RemoteObjectPipe, logger, log_to_stderr
 
 __all__ = ["RemoteObjectPipe", "start_tool", "Tool", "PipeShutdownError", "logger"]
@@ -32,7 +31,7 @@ def stop(self, pipe):
         pass
 
 
-def start_tool(pyenv, tools, group=None, name=None, logging=None):
+def start_tool(pyenv, tools, group=None, name=None, logging=None, tracing=None):
     """
     Starts a tool process
     """
@@ -42,7 +41,8 @@ def start_tool(pyenv, tools, group=None, name=None, logging=None):
         logger.setLevel(int(logging))
 
     pipe = RemoteObjectPipe.new_pipe()
-    proc = Process(pyenv, group=group, target=_instrumented_tool, name=name, args=(tools, pipe))
+    proc = Process(pyenv, group=group, name=name,
+                   target=_traced_tool if tracing else _instrumented_tool, args=(tools, pipe))
 
     try:
         proc.start()
@@ -53,6 +53,16 @@ def start_tool(pyenv, tools, group=None, name=None, logging=None):
     return proc, pipe
 
 
+def _traced_tool(tools, pipe):
+    import trace
+
+    def _print(*objects, sep=' ', end='', **kwargs):
+        logger.debug((sep.join(objects) + end).rstrip("\r\n"))
+
+    trace.print = _print
+    trace.Trace(count=0).runfunc(_instrumented_tool, tools, pipe)
+
+
 def _instrumented_tool(tools, pipe):
     try:
         for tool in tools:
@@ -69,4 +79,28 @@ def _instrumented_tool(tools, pipe):
     except Exception as e:
         pipe.close(e)
     finally:
-        pipe.close()
+        try:
+            pipe.close()
+        finally:
+            import threading
+
+            main = threading.main_thread()
+            current = threading.current_thread()
+
+            if main != current:
+                logger.warn("current thread %s is not the main %s in the tool process", current, main)
+
+            blocked_threads = False
+            for t in threading.enumerate():
+                if not t.daemon and t != current:
+                    logger.warn("non-daemon thread %s is blocking the tool process shutdown", t)
+                    blocked_threads = True
+
+            if blocked_threads:
+                import os
+                import atexit
+
+                try:
+                    atexit._run_exitfuncs()
+                finally:
+                    os._exit(1)

src/main/python/pybuilder/plugins/python/remote_tools/unittest_tool.py

@@ -56,6 +56,6 @@ def stop(self, pipe):
         pipe.hide("unittest_tests")
 
 
-def start_unittest_tool(pyenv, tools, sys_paths, test_modules, test_method_prefix, logging=0):
+def start_unittest_tool(pyenv, tools, sys_paths, test_modules, test_method_prefix, logging=0, tracing=0):
     tool = UnitTestTool(sys_paths, test_modules, test_method_prefix)
-    return start_tool(pyenv, tools + [tool], name="unittest", logging=logging)
+    return start_tool(pyenv, tools + [tool], name="unittest", logging=logging, tracing=tracing)

src/main/python/pybuilder/plugins/python/unittest_plugin.py

@@ -91,7 +91,8 @@ def run_tests(project, logger, reactor, execution_prefix, execution_name):
             reactor.python_env_registry[project.get_property("unittest_python_env")],
             reactor.tools, runner_generator, logger, test_dir, module_glob, [test_dir, src_dir],
             test_method_prefix,
-            project.get_property("remote_debug"))
+            project.get_property("remote_debug"),
+            project.get_property("remote_tracing"))
 
         if result.testsRun == 0:
             logger.warn("No %s executed.", execution_name)
@@ -121,23 +122,24 @@ def run_tests(project, logger, reactor, execution_prefix, execution_name):
 
 
 def execute_tests(pyenv, tools, runner_generator, logger, test_source, suffix, sys_paths, test_method_prefix=None,
-                  remote_debug=0):
+                  remote_debug=0, remote_tracing=0):
     return execute_tests_matching(pyenv, tools, runner_generator, logger, test_source, "*{0}".format(suffix),
-                                  test_method_prefix, remote_debug=remote_debug)
+                                  test_method_prefix, remote_debug=remote_debug, remote_tracing=remote_tracing)
 
 
 def execute_tests_matching(pyenv, tools, runner_generator, logger, test_source, file_glob, sys_paths,
-                           test_method_prefix=None, remote_debug=0):
+                           test_method_prefix=None, remote_debug=0, remote_tracing=0):
     output_log_file = StringIO()
     try:
         test_modules = discover_modules_matching(test_source, file_glob)
         runner = _instrument_runner(runner_generator,
                                     logger,
                                     _create_runner(runner_generator, output_log_file))
 
+        exit_code = None
         try:
             proc, pipe = start_unittest_tool(pyenv, tools, sys_paths, test_modules, test_method_prefix,
-                                             logging=remote_debug)
+                                             logging=remote_debug, tracing=remote_tracing)
             try:
                 pipe.register_remote(runner)
                 pipe.register_remote_type(unittest.result.TestResult)
@@ -153,9 +155,15 @@ def execute_tests_matching(pyenv, tools, runner_generator, logger, test_source,
                         proc.join()
                     finally:
                         try:
-                            proc.close()
-                        except AttributeError:
-                            pass
+                            exit_code = proc.exitcode
+                        finally:
+                            try:
+                                proc.close()
+                            except AttributeError:
+                                pass
+
+            if exit_code:
+                raise BuildFailedException("Unittest tool failed with exit code %s", exit_code)
 
             remote_closed_cause = pipe.remote_close_cause()
             if remote_closed_cause is not None: