Can't debug python with multiprocessing

[gramster]

@CitizenSanity commented on Fri Dec 20 2019

Using a simple python multiprocessing script like:

from multiprocessing.pool import Pool

def myFunc(funky_var):
    print("This is SPARTA!!!\n\t" + str(funky_var))

myPool = Pool()
lst = range(0,2)

myPool.map(myFunc,lst)
myPool.close()
myPool.join()

I expect my script to run~ but instead get a massive error chain:

root@998da5c4c021:/workspaces/Machine Learning# /usr/local/bin/python /root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/ptvsd_launcher.py --default --client --host localhost --port 44471 "/workspaces/Machine Learning/test.py" 
E00000.408: Exception escaped from start_client
            
            Traceback (most recent call last):
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/log.py", line 110, in g
                return f(*args, **kwargs)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/pydevd_hooks.py", line 74, in start_client
                sock, start_session = daemon.start_client((host, port))
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 214, in start_client
                with self.started():
              File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
                return next(self.gen)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 110, in started
                self.start()
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 145, in start
                raise RuntimeError('already started')
            RuntimeError: already started
            

Traceback (most recent call last):

E00000.419: Exception escaped from start_client
            
            Traceback (most recent call last):
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/log.py", line 110, in g
                return f(*args, **kwargs)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/pydevd_hooks.py", line 74, in start_client
                sock, start_session = daemon.start_client((host, port))
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 214, in start_client
                with self.started():
              File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
                return next(self.gen)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 110, in started
                self.start()
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 145, in start
                raise RuntimeError('already started')
            RuntimeError: already started
            

Traceback (most recent call last):


During handling of the above exception, another exception occurred:


E00000.423: Exception escaped from start_client
            
            Traceback (most recent call last):
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/log.py", line 110, in g
                return f(*args, **kwargs)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/pydevd_hooks.py", line 74, in start_client
                sock, start_session = daemon.start_client((host, port))
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 214, in start_client
                with self.started():
              File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
                return next(self.gen)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 110, in started
                self.start()
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 145, in start
                raise RuntimeError('already started')
            RuntimeError: already started
            

Traceback (most recent call last):


During handling of the above exception, another exception occurred:

E00000.426: Exception escaped from start_client
            
            Traceback (most recent call last):
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/log.py", line 110, in g
                return f(*args, **kwargs)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/pydevd_hooks.py", line 74, in start_client
                sock, start_session = daemon.start_client((host, port))
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 214, in start_client
                with self.started():
              File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
                return next(self.gen)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 110, in started
                self.start()
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 145, in start
                raise RuntimeError('already started')
            RuntimeError: already started
            


Traceback (most recent call last):


During handling of the above exception, another exception occurred:


E00000.430: Exception escaped from start_client
            
            Traceback (most recent call last):
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/log.py", line 110, in g
                return f(*args, **kwargs)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/pydevd_hooks.py", line 74, in start_client
                sock, start_session = daemon.start_client((host, port))
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 214, in start_client
                with self.started():
              File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
                return next(self.gen)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 110, in started
                self.start()
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 145, in start
                raise RuntimeError('already started')
            RuntimeError: already started
            

Error in atexit._run_exitfuncs:
Traceback (most recent call last):
Terminated
root@998da5c4c021:/workspaces/Machine Learning# 

During handling of the above exception, another exception occurred:


Error in atexit._run_exitfuncs:
Error in atexit._run_exitfuncs:
Traceback (most recent call last):
E00000.434: Exception escaped from start_client
            
            Traceback (most recent call last):
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/log.py", line 110, in g
                return f(*args, **kwargs)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/pydevd_hooks.py", line 74, in start_client
                sock, start_session = daemon.start_client((host, port))
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 214, in start_client
                with self.started():
              File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
                return next(self.gen)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 110, in started
                self.start()
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 145, in start
                raise RuntimeError('already started')
            RuntimeError: already started
            


Traceback (most recent call last):


During handling of the above exception, another exception occurred:


E00000.438: Exception escaped from start_client
            
            Traceback (most recent call last):
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/log.py", line 110, in g
                return f(*args, **kwargs)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/pydevd_hooks.py", line 74, in start_client
                sock, start_session = daemon.start_client((host, port))
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 214, in start_client
                with self.started():
              File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
                return next(self.gen)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 110, in started
                self.start()
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 145, in start
                raise RuntimeError('already started')
            RuntimeError: already started
            

Error in atexit._run_exitfuncs:
Traceback (most recent call last):
E00000.442: Exception escaped from start_client
            
            Traceback (most recent call last):
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/log.py", line 110, in g
                return f(*args, **kwargs)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/pydevd_hooks.py", line 74, in start_client
                sock, start_session = daemon.start_client((host, port))
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 214, in start_client
                with self.started():
              File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
                return next(self.gen)
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 110, in started
                self.start()
              File "/root/.vscode-server/extensions/ms-python.python-2019.11.50794/pythonFiles/lib/python/old_ptvsd/ptvsd/daemon.py", line 145, in start
                raise RuntimeError('already started')
            RuntimeError: already started
            

Traceback (most recent call last):


During handling of the above exception, another exception occurred:


Error in atexit._run_exitfuncs:
Error in atexit._run_exitfuncs:
Traceback (most recent call last):

Env: Python3 dev container

[karthiknadig]

@CitizenSanity see instructions here. microsoft/ptvsd#943 essentially, you may need to set the spawn as the set_start_method. This is only temporary until the new debugger becomes available to everybody.

def myFunc(funky_var):
    print("This is SPARTA!!!\n\t" + str(funky_var))

if __name__ == "__main__":
    import multiprocessing
    multiprocessing.set_start_method('spawn', True)
    from multiprocessing.pool import Pool    

    myPool = Pool()
    lst = range(0,2)

    myPool.map(myFunc,lst)
    myPool.close()
    myPool.join()

[zhangzp9970]

when debugging pytorch code, the extension will encounter the same issue

[karthiknadig]

@zhangzp9970 @CitizenSanity Try these instructions to enable the new debugger: microsoft/ptvsd#1706 (comment)

[ldurka]

A similar issue occurs when main process spawns subprocess.Popen an than Popen spawns multiprocessing.Process. In real life, this code exists in py-cpuinfo
What is important this issue occurs only with plugin Remote Development and debugging, without debugger it works fine.

Lunch configuration for the debugger is

{
    // Use IntelliSense to learn about possible attributes.
    // Hover to view descriptions of existing attributes.
    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
    "version": "0.2.0",
    "configurations": [
        {
            "name": "Python: Current File",
            "type": "python",
            "request": "launch",
            "program": "${file}",
            "console": "integratedTerminal",
            "subProcess": true,
            "justMyCode": false
        }
    ]
}

Code example with cpuinfo

from cpuinfo import get_cpu_info
z = get_cpu_info()
# and iside the 
# get_cpu_info() -> 
# main() ->
# _get_cpu_info_from_cpuid
# Wait for the process to end, while it is still alive
# while p.is_alive():
#     p.join(0) <- and debugger stuck on this and process never ends

Another synthetic example also occurs only with Remote Development

import sys
from multiprocessing import Process
from subprocess import PIPE, Popen


# Similar code is in cpuinfo/cpuinfo.py
def main():
    print("main")
    command = ["python", __file__, "some", "extra", "args"]

    p = Popen(command, stdout=PIPE, stderr=PIPE, stdin=PIPE)
    print("spawn subprocess_popen")
    output = p.communicate()[0]
    return p.returncode, output


def multiprocessing_process():
    print("spawned multiprocessing_process")


def subprocess_popen():
    print("subprocess_popen")
    p = Process(target=multiprocessing_process)
    p.start()

    # Wait for the process to end, while it is still alive
    # and the VS Debugger stuck here
    while p.is_alive():
        p.join(0)

    # Return {} if it failed
    if p.exitcode != 0:
        return


if __name__ == "__main__":
    if len(sys.argv) > 1:
        subprocess_popen()
    else:
        main()

[karthiknadig]

@ldurka We have addressed this in the latest insiders. The issue was that on remote development scenario we did not pickup the setting to enable the new debugger that supports multiprocessing. Please try it out. You will need to update the version of the extension installed in the remote environment. See instructions here: microsoft/debugpy#59

[karthiknadig]

Closing this issue since we have addressed this. For anyone running into this issue please follow instructions here: microsoft/debugpy#59