Subprocesses should inherit PydevdCustomization. Fixes #1874

src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_monkey.py

@@ -36,14 +36,44 @@ def _get_apply_arg_patching():
     return getattr(_arg_patch, 'apply_arg_patching', True)
 
 
+def _get_setup_updated_with_protocol(setup):
+    if setup is None:
+        setup = {}
+    setup = setup.copy()
+    # Discard anything related to the protocol (we'll set the the protocol based on the one
+    # currently set).
+    setup.pop(pydevd_constants.ARGUMENT_HTTP_JSON_PROTOCOL, None)
+    setup.pop(pydevd_constants.ARGUMENT_JSON_PROTOCOL, None)
+    setup.pop(pydevd_constants.ARGUMENT_QUOTED_LINE_PROTOCOL, None)
+    setup.pop(pydevd_constants.ARGUMENT_HTTP_PROTOCOL, None)
+
+    protocol = pydevd_constants.get_protocol()
+    if protocol == pydevd_constants.HTTP_JSON_PROTOCOL:
+        setup[pydevd_constants.ARGUMENT_HTTP_JSON_PROTOCOL] = True
+
+    elif protocol == pydevd_constants.JSON_PROTOCOL:
+        setup[pydevd_constants.ARGUMENT_JSON_PROTOCOL] = True
+
+    elif protocol == pydevd_constants.QUOTED_LINE_PROTOCOL:
+        setup[pydevd_constants.ARGUMENT_QUOTED_LINE_PROTOCOL] = True
+
+    elif protocol == pydevd_constants.HTTP_PROTOCOL:
+        setup[pydevd_constants.ARGUMENT_HTTP_PROTOCOL] = True
+
+    else:
+        pydev_log.debug('Unexpected protocol: %s', protocol)
+    return setup
+
+
 def _get_python_c_args(host, port, indC, args, setup):
-    host_literal = "'" + host + "'" if host is not None else 'None'
-    return ("import sys; sys.path.append(r'%s'); import pydevd; "
-            "pydevd.settrace(host=%s, port=%s, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True); "
+    setup = _get_setup_updated_with_protocol(setup)
+    return ("import sys; sys.path.append(r'%s'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL=%r;"
+            "pydevd.settrace(host=%r, port=%s, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True); "
             "from pydevd import SetupHolder; SetupHolder.setup = %s; %s"
             ) % (
                pydev_src_dir,
-               host_literal,
+               pydevd_constants.get_protocol(),
+               host,
                port,
                setup,
                args[indC + 1])
@@ -210,7 +240,7 @@ def patch_args(args):
         # ['X:\\pysrc\\pydevd.py', '--multiprocess', '--print-in-debugger-startup',
         #  '--vm_type', 'python', '--client', '127.0.0.1', '--port', '56352', '--file', 'x:\\snippet1.py']
         from _pydevd_bundle.pydevd_command_line_handling import setup_to_argv
-        original = setup_to_argv(SetupHolder.setup) + ['--file']
+        original = setup_to_argv(_get_setup_updated_with_protocol(SetupHolder.setup)) + ['--file']
         while i < len(args):
             if args[i] == '-m':
                 # Always insert at pos == 1 (i.e.: pydevd "--module" --multiprocess ...)

src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_command_line_handling.py

@@ -65,8 +65,13 @@ def handle_argv(self, argv, i, setup):
     ArgHandlerBool('print-in-debugger-startup'),
     ArgHandlerBool('cmd-line'),
     ArgHandlerBool('module'),
+
+    # The ones below should've been just one setting to specify the protocol, but for compatibility
+    # reasons they're passed as a flag but are mutually exclusive.
     ArgHandlerBool('json-dap'),  # Protocol used by ptvsd to communicate with pydevd (a single json message in each read)
     ArgHandlerBool('json-dap-http'),  # Actual DAP (json messages over http protocol).
+    ArgHandlerBool('protocol-quoted-line'),  # Custom protocol with quoted lines.
+    ArgHandlerBool('protocol-http'),  # Custom protocol with http.
 ]
 
 ARGV_REP_TO_HANDLER = {}

src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py

@@ -586,19 +586,23 @@ def new_func(*args, **kwargs):
 # Protocol where each line is a new message (text is quoted to prevent new lines).
 # payload is xml
 QUOTED_LINE_PROTOCOL = 'quoted-line'
+ARGUMENT_QUOTED_LINE_PROTOCOL = 'protocol-quoted-line'
 
 # Uses http protocol to provide a new message.
 # i.e.: Content-Length:xxx\r\n\r\npayload
 # payload is xml
 HTTP_PROTOCOL = 'http'
+ARGUMENT_HTTP_PROTOCOL = 'protocol-http'
 
 # Message is sent without any header.
 # payload is json
 JSON_PROTOCOL = 'json'
+ARGUMENT_JSON_PROTOCOL = 'json-dap'
 
 # Same header as the HTTP_PROTOCOL
 # payload is json
 HTTP_JSON_PROTOCOL = 'http_json'
+ARGUMENT_HTTP_JSON_PROTOCOL = 'json-dap-http'
 
 
 class _GlobalSettings:

src/ptvsd/_vendored/pydevd/pydevd.py

@@ -25,7 +25,7 @@
 from _pydev_imps._pydev_saved_modules import thread
 from _pydev_imps._pydev_saved_modules import threading
 from _pydev_imps._pydev_saved_modules import time
-from _pydevd_bundle import pydevd_extension_utils, pydevd_frame_utils
+from _pydevd_bundle import pydevd_extension_utils, pydevd_frame_utils, pydevd_constants
 from _pydevd_bundle.pydevd_filtering import FilesFiltering
 from _pydevd_bundle import pydevd_io, pydevd_vm_type
 from _pydevd_bundle import pydevd_utils
@@ -41,7 +41,7 @@
     clear_cached_thread_id, INTERACTIVE_MODE_AVAILABLE, SHOW_DEBUG_INFO_ENV, IS_PY34_OR_GREATER, IS_PY2, NULL,
     NO_FTRACE, IS_IRONPYTHON, JSON_PROTOCOL, IS_CPYTHON, HTTP_JSON_PROTOCOL, USE_CUSTOM_SYS_CURRENT_FRAMES_MAP, call_only_once,
     ForkSafeLock)
-from _pydevd_bundle.pydevd_defaults import PydevdCustomization
+from _pydevd_bundle.pydevd_defaults import PydevdCustomization  # Note: import alias used on pydev_monkey.
 from _pydevd_bundle.pydevd_custom_frames import CustomFramesContainer, custom_frames_container_init
 from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE, PYDEV_FILE, LIB_FILE
 from _pydevd_bundle.pydevd_extension_api import DebuggerEventHandler
@@ -2910,6 +2910,7 @@ def getpass(*args, **kwargs):
 def main():
 
     # parse the command line. --file is our last argument that is required
+    pydev_log.debug("Initial arguments: %s", (sys.argv,))
     try:
         from _pydevd_bundle.pydevd_command_line_handling import process_command_line
         setup = process_command_line(sys.argv)
@@ -2925,8 +2926,8 @@ def main():
             pid = ''
         sys.stderr.write("pydev debugger: starting%s\n" % pid)
 
-    pydev_log.debug("Executing file %s" % setup['file'])
-    pydev_log.debug("arguments: %s" % str(sys.argv))
+    pydev_log.debug("Executing file %s", setup['file'])
+    pydev_log.debug("arguments: %s", (sys.argv,))
 
     pydevd_vm_type.setup_type(setup.get('vm_type', None))
 
@@ -2964,7 +2965,7 @@ def main():
                 dispatcher.connect(host, port)
                 if dispatcher.port is not None:
                     port = dispatcher.port
-                    pydev_log.debug("Received port %d\n" % port)
+                    pydev_log.debug("Received port %d\n", port)
                     pydev_log.info("pydev debugger: process %d is connecting\n" % os.getpid())
 
                     try:
@@ -3036,12 +3037,18 @@ def main():
     is_module = setup['module']
     patch_stdin()
 
-    if setup['json-dap']:
+    if setup[pydevd_constants.ARGUMENT_JSON_PROTOCOL]:
         PyDevdAPI().set_protocol(debugger, 0, JSON_PROTOCOL)
 
-    elif setup['json-dap-http']:
+    elif setup[pydevd_constants.ARGUMENT_HTTP_JSON_PROTOCOL]:
         PyDevdAPI().set_protocol(debugger, 0, HTTP_JSON_PROTOCOL)
 
+    elif setup[pydevd_constants.ARGUMENT_HTTP_PROTOCOL]:
+        PyDevdAPI().set_protocol(debugger, 0, pydevd_constants.HTTP_PROTOCOL)
+
+    elif setup[pydevd_constants.ARGUMENT_QUOTED_LINE_PROTOCOL]:
+        PyDevdAPI().set_protocol(debugger, 0, pydevd_constants.QUOTED_LINE_PROTOCOL)
+
     access_token = setup['access-token']
     if access_token:
         debugger.authentication.access_token = access_token

src/ptvsd/_vendored/pydevd/tests_python/debugger_fixtures.py

@@ -324,6 +324,7 @@ def test_file(
                 wait_for_port=True,
                 access_token=None,
                 ide_access_token=None,
+                append_command_line_args=(),
                 **kwargs
             ):
 
@@ -338,6 +339,8 @@ def update_command_line_args(writer, args):
                 if ide_access_token is not None:
                     ret.append('--ide-access-token')
                     ret.append(ide_access_token)
+
+                ret.extend(append_command_line_args)
                 return ret
 
             WriterThread.TEST_FILE = debugger_unittest._get_debugger_test_file(filename)

src/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py

@@ -459,6 +459,7 @@ def create_process(self, args, writer):
 
         env['PYDEVD_DEBUG'] = 'True'
         env['PYDEVD_DEBUG_FILE'] = self.pydevd_debug_file
+        print('Logging to: %s' % (self.pydevd_debug_file,))
         process = subprocess.Popen(
             args,
             stdout=subprocess.PIPE,

src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_pydevd_customization.py

@@ -0,0 +1,51 @@
+import sys
+import os
+
+
+def main():
+    env = os.environ.copy()
+    pythonpath = env.get('PYTHONPATH', '')
+    env['PYTHONPATH'] = os.path.dirname(__file__) + os.pathsep + \
+        os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+
+    from _pydevd_bundle.pydevd_constants import HTTP_JSON_PROTOCOL
+    from _pydevd_bundle.pydevd_defaults import PydevdCustomization
+    PydevdCustomization.DEFAULT_PROTOCOL = HTTP_JSON_PROTOCOL
+
+    import pydevd
+    from _pydev_bundle import pydev_log
+    pydev_log.debug('Argv received: %s', sys.argv)
+    port = int(sys.argv[1])
+    print('before pydevd.settrace')
+    pydevd.settrace(port=port, patch_multiprocessing=True, suspend=True)
+    print('after pydevd.settrace')
+
+    import subprocess
+    if '--use-c-switch' in sys.argv:
+        p = subprocess.Popen(
+            [sys.executable, '-u', '-c', 'import _debugger_case_pydevd_customization;_debugger_case_pydevd_customization.call()'],
+            stdout=subprocess.PIPE,
+            env=env,
+        )
+    else:
+        p = subprocess.Popen(
+            [sys.executable, '-u', '_debugger_case_pydevd_customization.py', '--simple-call'],
+            cwd=os.path.dirname(__file__),
+            stdout=subprocess.PIPE,
+            env=env,
+        )
+
+    stdout, stderr = p.communicate()
+    assert b'called' in stdout, 'Did not find b"called" in: %s' % (stdout,)
+    print('TEST SUCEEDED!')  # break 2 here
+
+
+def call():
+    print("called")  # break 1 here
+
+
+if __name__ == '__main__':
+    if '--simple-call' in sys.argv:
+        call()
+    else:
+        main()

src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py

@@ -2961,6 +2961,96 @@ def test_attach_to_pid(case_setup_remote, reattach):
         writer.finished_ok = True
 
 
+def test_remote_debugger_basic(case_setup_remote):
+    with case_setup_remote.test_file('_debugger_case_remote.py') as writer:
+        json_facade = JsonFacade(writer)
+        json_facade.write_launch()
+        json_facade.write_make_initial_run()
+        json_facade.wait_for_thread_stopped()
+        json_facade.write_continue()
+
+        writer.finished_ok = True
+
+
+@pytest.mark.parametrize('use_c_switch', [True, False])
+def test_subprocess_pydevd_customization(case_setup_remote, use_c_switch):
+    import threading
+    from tests_python.debugger_unittest import AbstractWriterThread
+
+    with case_setup_remote.test_file(
+            '_debugger_case_pydevd_customization.py',
+            append_command_line_args=['--use-c-switch'] if use_c_switch else []
+        ) as writer:
+        json_facade = JsonFacade(writer, send_json_startup_messages=False)
+        json_facade.writer.write_multi_threads_single_notification(True)
+        json_facade.write_launch()
+
+        break1_line = writer.get_line_index_with_content('break 1 here')
+        break2_line = writer.get_line_index_with_content('break 2 here')
+        json_facade.write_set_breakpoints([break1_line, break2_line])
+
+        server_socket = writer.server_socket
+
+        class SecondaryProcessWriterThread(AbstractWriterThread):
+
+            TEST_FILE = writer.get_main_filename()
+            _sequence = -1
+
+        class SecondaryProcessThreadCommunication(threading.Thread):
+
+            def run(self):
+                from tests_python.debugger_unittest import ReaderThread
+                expected_connections = 1
+                if sys.platform != 'win32' and IS_PY2:
+                    # Note: on linux on Python 2 CPython subprocess.call will actually
+                    # create a fork first (at which point it'll connect) and then, later on it'll
+                    # call the main (as if it was a clean process as if PyDB wasn't created
+                    # the first time -- the debugger will still work, but it'll do an additional
+                    # connection).
+                    expected_connections = 2
+
+                for _ in range(expected_connections):
+                    server_socket.listen(1)
+                    self.server_socket = server_socket
+                    writer.log.append('  *** Multiprocess waiting on server_socket.accept()')
+                    new_sock, addr = server_socket.accept()
+                    writer.log.append('  *** Multiprocess completed server_socket.accept()')
+
+                    reader_thread = ReaderThread(new_sock)
+                    reader_thread.name = '  *** Multiprocess Reader Thread'
+                    reader_thread.start()
+                    writer.log.append('  *** Multiprocess started ReaderThread')
+
+                    writer2 = SecondaryProcessWriterThread()
+                    writer2._WRITE_LOG_PREFIX = '  *** Multiprocess write: '
+                    writer2.reader_thread = reader_thread
+                    writer2.sock = new_sock
+                    json_facade2 = JsonFacade(writer2, send_json_startup_messages=False)
+                    json_facade2.writer.write_multi_threads_single_notification(True)
+
+                    json_facade2.write_set_breakpoints([break1_line, break2_line])
+                    json_facade2.write_make_initial_run()
+
+                json_facade2.wait_for_thread_stopped()
+                json_facade2.write_continue()
+
+        secondary_process_thread_communication = SecondaryProcessThreadCommunication()
+        secondary_process_thread_communication.start()
+        time.sleep(.1)
+
+        json_facade.write_make_initial_run()
+        json_facade.wait_for_thread_stopped()
+
+        json_facade.write_continue()
+        json_facade.wait_for_thread_stopped()
+        json_facade.write_continue()
+
+        secondary_process_thread_communication.join(5)
+        if secondary_process_thread_communication.is_alive():
+            raise AssertionError('The SecondaryProcessThreadCommunication did not finish')
+        writer.finished_ok = True
+
+
 @pytest.mark.parametrize('apply_multiprocessing_patch', [True, False])
 def test_no_subprocess_patching(case_setup_multiprocessing, apply_multiprocessing_patch):
     import threading

src/ptvsd/_vendored/pydevd/tests_python/test_pydev_monkey.py

@@ -17,12 +17,13 @@ def test_monkey(self):
         original = SetupHolder.setup
 
         try:
-            SetupHolder.setup = {'client': '127.0.0.1', 'port': '0'}
+            SetupHolder.setup = {'client': '127.0.0.1', 'port': '0', 'protocol-quoted-line': True}
             check = '''C:\\bin\\python.exe -u -c connect(\\"127.0.0.1\\")'''
             debug_command = (
                 'import sys; '
                 'sys.path.append(r\'%s\'); '
-                "import pydevd; pydevd.settrace(host='127.0.0.1', port=0, suspend=False, "
+                "import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL='quoted-line';"
+                "pydevd.settrace(host='127.0.0.1', port=0, suspend=False, "
                 'trace_only_current_thread=False, patch_multiprocessing=True); '
                 ''
                 "from pydevd import SetupHolder; "
@@ -46,10 +47,10 @@ def test_monkey_patch_args_indc(self):
         original = SetupHolder.setup
 
         try:
-            SetupHolder.setup = {'client': '127.0.0.1', 'port': '0'}
+            SetupHolder.setup = {'client': '127.0.0.1', 'port': '0', 'protocol-quoted-line': True}
             check = ['C:\\bin\\python.exe', '-u', '-c', 'connect("127.0.0.1")']
             debug_command = (
-                'import sys; sys.path.append(r\'%s\'); import pydevd; '
+                "import sys; sys.path.append(r\'%s\'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL='quoted-line';"
                 'pydevd.settrace(host=\'127.0.0.1\', port=0, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True); '
                 ''
                 "from pydevd import SetupHolder; "
@@ -85,6 +86,7 @@ def test_monkey_patch_args_module(self):
                 '--client',
                 '127.0.0.1',
                 '--multiprocess',
+                '--protocol-quoted-line',
                 '--file',
                 'test',
             ])
@@ -105,6 +107,7 @@ def test_monkey_patch_args_no_indc(self):
                 '0',
                 '--client',
                 '127.0.0.1',
+                '--protocol-quoted-line',
                 '--file',
                 '"connect(\\\\\\"127.0.0.1\\\\\\")"' if sys.platform == 'win32' else 'connect(\\"127.0.0.1\\")',
                 '"with spaces"'  if sys.platform == 'win32' else 'with spaces',
@@ -138,6 +141,7 @@ def test_monkey_patch_args_no_indc_without_pydevd(self):
                 '0',
                 '--client',
                 '127.0.0.1',
+                '--protocol-quoted-line',
                 '--file',
                 'target.py',
                 '"connect(\\\\\\"127.0.0.1\\\\\\")"' if sys.platform == 'win32' else 'connect(\\"127.0.0.1\\")',
@@ -162,6 +166,7 @@ def test_monkey_patch_c_program_arg(self):
                 '0',
                 '--client',
                 '127.0.0.1',
+                '--protocol-quoted-line',
                 '--file',
                 'target.py',
                 '-c',