add support for os.fork and related functions (RustPython#4877)

Author: itsankitkp

Lib/concurrent/futures/thread.py

@@ -37,7 +37,8 @@ def _python_exit():
 threading._register_atexit(_python_exit)
 
 # At fork, reinitialize the `_global_shutdown_lock` lock in the child process
-if hasattr(os, 'register_at_fork'):
+# TODO RUSTPYTHON - _at_fork_reinit is not implemented yet
+if hasattr(os, 'register_at_fork') and hasattr(_global_shutdown_lock, '_at_fork_reinit'):
     os.register_at_fork(before=_global_shutdown_lock.acquire,
                         after_in_child=_global_shutdown_lock._at_fork_reinit,
                         after_in_parent=_global_shutdown_lock.release)

Lib/test/test_builtin.py

@@ -37,6 +37,8 @@
 except ImportError:
     pty = signal = None
 
+import threading  # XXX: RUSTPYTHON; to skip _at_fork_reinit
+
 
 class Squares:
 
@@ -2269,6 +2271,7 @@ def test_input_tty_non_ascii_unicode_errors(self):
         # Check stdin/stdout error handler is used when invoking PyOS_Readline()
         self.check_input_tty("prompté", b"quux\xe9", "ascii")
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_input_no_stdout_fileno(self):
         # Issue #24402: If stdin is the original terminal but stdout.fileno()
         # fails, do not use the original stdout file descriptor

Lib/test/test_fcntl.py

@@ -4,6 +4,7 @@
 import os
 import struct
 import sys
+import threading  # XXX: RUSTPYTHON
 import unittest
 from multiprocessing import Process
 from test.support import verbose, cpython_only
@@ -155,8 +156,9 @@ def test_flock(self):
         self.assertRaises(ValueError, fcntl.flock, -1, fcntl.LOCK_SH)
         self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH)
 
-    # TODO: RUSTPYTHON, AttributeError: module 'os' has no attribute 'fork'
+    # TODO: RUSTPYTHON
     @unittest.expectedFailure
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError")
     def test_lockf_exclusive(self):
         self.f = open(TESTFN, 'wb+')
@@ -167,9 +169,9 @@ def test_lockf_exclusive(self):
         p.join()
         fcntl.lockf(self.f, fcntl.LOCK_UN)
         self.assertEqual(p.exitcode, 0)
-
-    # TODO: RUSTPYTHON, AttributeError: module 'os' has no attribute 'fork'
+    # TODO: RUSTPYTHON
     @unittest.expectedFailure
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError")
     def test_lockf_share(self):
         self.f = open(TESTFN, 'wb+')

Lib/test/test_httpservers.py

@@ -607,7 +607,8 @@ def test_html_escape_filename(self):
 print("</pre>")
 """
 
-
+@unittest.skipIf(not hasattr(os, '_exit'),
+        "TODO: RUSTPYTHON, run_cgi in http/server.py gets stuck as os._exit(127) doesn't currently kill forked processes")
 @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
         "This test can't be run reliably as root (issue #13308).")
 class CGIHTTPServerTestCase(BaseTestCase):
@@ -765,6 +766,7 @@ def test_url_collapse_path(self):
                                  msg='path = %r\nGot:    %r\nWanted: %r' %
                                  (path, actual, expected))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_headers_and_content(self):
         res = self.request('/cgi-bin/file1.py')
         self.assertEqual(
@@ -775,6 +777,7 @@ def test_issue19435(self):
         res = self.request('///////////nocgi.py/../cgi-bin/nothere.sh')
         self.assertEqual(res.status, HTTPStatus.NOT_FOUND)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_post(self):
         params = urllib.parse.urlencode(
             {'spam' : 1, 'eggs' : 'python', 'bacon' : 123456})
@@ -788,6 +791,7 @@ def test_invaliduri(self):
         res.read()
         self.assertEqual(res.status, HTTPStatus.NOT_FOUND)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_authorization(self):
         headers = {b'Authorization' : b'Basic ' +
                    base64.b64encode(b'username:pass')}
@@ -796,13 +800,15 @@ def test_authorization(self):
             (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_no_leading_slash(self):
         # http://bugs.python.org/issue2254
         res = self.request('cgi-bin/file1.py')
         self.assertEqual(
             (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_os_environ_is_not_altered(self):
         signature = "Test CGI Server"
         os.environ['SERVER_SOFTWARE'] = signature
@@ -812,31 +818,36 @@ def test_os_environ_is_not_altered(self):
             (res.read(), res.getheader('Content-type'), res.status))
         self.assertEqual(os.environ['SERVER_SOFTWARE'], signature)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_urlquote_decoding_in_cgi_check(self):
         res = self.request('/cgi-bin%2ffile1.py')
         self.assertEqual(
             (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_nested_cgi_path_issue21323(self):
         res = self.request('/cgi-bin/child-dir/file3.py')
         self.assertEqual(
             (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_query_with_multiple_question_mark(self):
         res = self.request('/cgi-bin/file4.py?a=b?c=d')
         self.assertEqual(
             (b'a=b?c=d' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_query_with_continuous_slashes(self):
         res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//')
         self.assertEqual(
             (b'k=aa%2F%2Fbb&//q//p//=//a//b//' + self.linesep,
              'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_cgi_path_in_sub_directories(self):
         try:
             CGIHTTPRequestHandler.cgi_directories.append('/sub/dir/cgi-bin')
@@ -847,6 +858,7 @@ def test_cgi_path_in_sub_directories(self):
         finally:
             CGIHTTPRequestHandler.cgi_directories.remove('/sub/dir/cgi-bin')
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     def test_accept(self):
         browser_accept = \
                     'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'

Lib/test/test_os.py

@@ -69,6 +69,10 @@
 except ImportError:
     INT_MAX = PY_SSIZE_T_MAX = sys.maxsize
 
+try:
+    import _testcapi
+except ImportError:
+    _testcapi = None
 
 from test.support.script_helper import assert_python_ok
 from test.support import unix_shell
@@ -3067,11 +3071,13 @@ def check_waitpid(self, code, exitcode, callback=None):
         self.assertEqual(pid2, pid)
 
     # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv')
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.expectedFailure
     def test_waitpid(self):
         self.check_waitpid(code='pass', exitcode=0)
 
     # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv')
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.expectedFailure
     def test_waitstatus_to_exitcode(self):
         exitcode = 23
@@ -3103,7 +3109,8 @@ def test_waitstatus_to_exitcode_windows(self):
             os.waitstatus_to_exitcode((max_exitcode + 1) << 8)
         with self.assertRaises(OverflowError):
             os.waitstatus_to_exitcode(-1)
-
+            
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv')
     @unittest.expectedFailure
     # Skip the test on Windows
@@ -3146,31 +3153,36 @@ def create_args(self, *, with_env=False, use_bytes=False):
                         for k, v in self.env.items()}
 
         return args
-
+    
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnl')
     def test_spawnl(self):
         args = self.create_args()
         exitcode = os.spawnl(os.P_WAIT, args[0], *args)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnle')
     def test_spawnle(self):
         args = self.create_args(with_env=True)
         exitcode = os.spawnle(os.P_WAIT, args[0], *args, self.env)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnlp')
     def test_spawnlp(self):
         args = self.create_args()
         exitcode = os.spawnlp(os.P_WAIT, args[0], *args)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnlpe')
     def test_spawnlpe(self):
         args = self.create_args(with_env=True)
         exitcode = os.spawnlpe(os.P_WAIT, args[0], *args, self.env)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnv')
     def test_spawnv(self):
         args = self.create_args()
@@ -3181,30 +3193,35 @@ def test_spawnv(self):
         exitcode = os.spawnv(os.P_WAIT, FakePath(args[0]), args)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnve')
     def test_spawnve(self):
         args = self.create_args(with_env=True)
         exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnvp')
     def test_spawnvp(self):
         args = self.create_args()
         exitcode = os.spawnvp(os.P_WAIT, args[0], args)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnvpe')
     def test_spawnvpe(self):
         args = self.create_args(with_env=True)
         exitcode = os.spawnvpe(os.P_WAIT, args[0], args, self.env)
         self.assertEqual(exitcode, self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnv')
     def test_nowait(self):
         args = self.create_args()
         pid = os.spawnv(os.P_NOWAIT, args[0], args)
         support.wait_process(pid, exitcode=self.exitcode)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnve')
     def test_spawnve_bytes(self):
         # Test bytes handling in parse_arglist and parse_envlist (#28114)
@@ -3286,10 +3303,12 @@ def _test_invalid_env(self, spawn):
         exitcode = spawn(os.P_WAIT, args[0], args, newenv)
         self.assertEqual(exitcode, 0)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnve')
     def test_spawnve_invalid_env(self):
         self._test_invalid_env(os.spawnve)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @requires_os_func('spawnvpe')
     def test_spawnvpe_invalid_env(self):
         self._test_invalid_env(os.spawnvpe)
@@ -4660,6 +4679,35 @@ def test_fork(self):
         assert_python_ok("-c", code)
         assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug")
 
+    @unittest.skipIf(_testcapi is None, 'TODO: RUSTPYTHON; needs _testcapi')
+    @unittest.skipUnless(sys.platform in ("linux", "darwin"),
+                         "Only Linux and macOS detect this today.")
+    def test_fork_warns_when_non_python_thread_exists(self):
+        code = """if 1:
+            import os, threading, warnings
+            from _testcapi import _spawn_pthread_waiter, _end_spawned_pthread
+            _spawn_pthread_waiter()
+            try:
+                with warnings.catch_warnings(record=True) as ws:
+                    warnings.filterwarnings(
+                            "always", category=DeprecationWarning)
+                    if os.fork() == 0:
+                        assert not ws, f"unexpected warnings in child: {ws}"
+                        os._exit(0)  # child
+                    else:
+                        assert ws[0].category == DeprecationWarning, ws[0]
+                        assert 'fork' in str(ws[0].message), ws[0]
+                        # Waiting allows an error in the child to hit stderr.
+                        exitcode = os.wait()[1]
+                        assert exitcode == 0, f"child exited {exitcode}"
+                assert threading.active_count() == 1, threading.enumerate()
+            finally:
+                _end_spawned_pthread()
+        """
+        _, out, err = assert_python_ok("-c", code, PYTHONOPTIMIZE='0')
+        self.assertEqual(err.decode("utf-8"), "")
+        self.assertEqual(out.decode("utf-8"), "")
+
 
 # Only test if the C version is provided, otherwise TestPEP519 already tested
 # the pure Python implementation.

Lib/test/test_platform.py

@@ -346,8 +346,6 @@ def test_mac_ver(self):
             else:
                 self.assertEqual(res[2], 'PowerPC')
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     @unittest.skipUnless(sys.platform == 'darwin', "OSX only test")
     def test_mac_ver_with_fork(self):
         # Issue7895: platform.mac_ver() crashes when using fork without exec

Lib/test/test_pty.py

@@ -1,3 +1,4 @@
+import threading  # XXX: RUSTPYTHON
 from test.support import verbose, reap_children
 from test.support.import_helper import import_module
 
@@ -211,6 +212,7 @@ def test_openpty(self):
         self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2))
 
     # TODO: RUSTPYTHON
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.expectedFailure
     def test_fork(self):
         debug("calling pty.fork()")
@@ -314,6 +316,7 @@ def test_master_read(self):
         self.assertEqual(data, b"")
 
     # TODO: RUSTPYTHON; no os.fork
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.expectedFailure
     def test_spawn_doesnt_hang(self):
         pty.spawn([sys.executable, '-c', 'print("hi there")'])

Lib/test/test_socketserver.py

@@ -55,7 +55,7 @@ class ForkingUnixDatagramServer(socketserver.ForkingMixIn,
                                     socketserver.UnixDatagramServer):
         pass
 
-
+@test.support.requires_fork() # TODO: RUSTPYTHON, os.fork is currently only supported on Unix-based systems
 @contextlib.contextmanager
 def simple_subprocess(testcase):
     """Tests that a custom child process is not waited on (Issue 1540386)"""

Lib/test/test_support.py

@@ -9,6 +9,7 @@
 import sys
 import tempfile
 import textwrap
+import threading  # XXX: RUSTPYTHON
 import time
 import unittest
 import warnings
@@ -453,6 +454,7 @@ def test_check__all__(self):
 
     # TODO: RUSTPYTHON
     @unittest.expectedFailure
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG'),
                          'need os.waitpid() and os.WNOHANG')
     @support.requires_fork()

Lib/test/test_tempfile.py

@@ -6,6 +6,7 @@
 import pathlib
 import sys
 import re
+import threading  # XXX: RUSTPYTHON; to check `_at_fork_reinit`
 import warnings
 import contextlib
 import stat
@@ -198,6 +199,7 @@ def supports_iter(self):
             if i == 20:
                 break
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(hasattr(os, 'fork'),
         "os.fork is required for this test")
     def test_process_awareness(self):
@@ -465,6 +467,7 @@ def test_file_mode(self):
             expected = user * (1 + 8 + 64)
         self.assertEqual(mode, expected)
 
+    @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit')
     @unittest.skipUnless(has_spawnl, 'os.spawnl not available')
     def test_noinherit(self):
         # _mkstemp_inner file handles are not inherited by child processes