[python] Fix multiprocessing pool on Python 3.8

Processes are now spawned instead of forked, which can cause lockups and
lost processes

test/all/run_all.py

@@ -20,11 +20,16 @@
 import random
 import tempfile
 import argparse
+import platform
 import subprocess
 import multiprocessing
 from pathlib import Path
 from collections import defaultdict
 
+# Set the process create method to fork rather than spawn
+if platform.system() != "Windows":
+    _ = multiprocessing.get_context("fork")
+
 def repopath(path):
     return Path(__file__).absolute().parents[2] / path
 

tools/scripts/examples_compile.py

@@ -16,6 +16,11 @@
 import multiprocessing
 from pathlib import Path
 
+# Set the process create method to fork rather than spawn
+if platform.system() != "Windows":
+    _ = multiprocessing.get_context("fork")
+
+
 is_running_in_ci = (os.getenv("CIRCLECI") is not None or
                     os.getenv("TRAVIS") is not None or
                     os.getenv("GITHUB_ACTIONS") is not None)
@@ -110,20 +115,23 @@ def compile_examples(paths, jobs, split, part):
     # first generate all projects
     with multiprocessing.Pool(jobs) as pool:
         projects = pool.map(generate, projects)
+    # projects = [generate(p) for p in projects]
     results += projects.count(None)
 
     # Filter projects for successful generation
     projects = [p for p in projects if p is not None]
     # Then build the successfully generated ones
     with multiprocessing.Pool(jobs) as pool:
         projects = pool.map(build, projects)
+    # projects = [build(p) for p in projects]
     results += projects.count(None)
 
     # Filter projects for successful compilation and runablity
     projects = [p for p in projects if p is not None and "CI: run" in p.read_text()]
     # Then run the successfully compiled ones
     with multiprocessing.Pool(jobs) as pool:
         projects = pool.map(run, projects)
+    # projects = [run(p) for p in projects]
     results += projects.count(None)
 
     return results