Fix remaining tests to reflect the debug adapter refactoring changes.

Fix Flask and Django multiprocess tests.

Fix test logs not being captured by pytest.

Fix "import debug_me" check improperly applied in tests where it is unnecessary.

Fix some clarifying patterns not respecting the underlying pattern.

Add pattern helpers for strings: starting_with, ending_with, containing.

Move DAP test helpers to a separate module, and add a helper for frames.

Add support for line markers when setting breakpoints and matching frames.

Assorted test fixes around handling of Unicode and paths.

.travis.yml

@@ -26,4 +26,4 @@ install:
   - pip install -U pip setuptools tox tox-travis
 
 script:
-  - tox
+  - tox -- -n4 -v

pytest.ini

@@ -2,4 +2,4 @@
 testpaths=tests
 timeout=30
 timeout_method=thread
-addopts=-n6
+addopts=-n8

src/ptvsd/common/compat.py

@@ -71,7 +71,7 @@ def force_bytes(s, encoding, errors="strict"):
         return s
 
 
-def force_str(s, encoding, errors="strict"):
+def force_str(s, encoding="ascii", errors="strict"):
     """Converts s to str (which is bytes on Python 2, and unicode on Python 3), using
     the provided encoding if necessary. If s is already str, it is returned as is.
 
@@ -137,6 +137,12 @@ def nameof(obj, quote=False):
     return force_unicode(name, "utf-8", "replace")
 
 
+def unicode_repr(obj):
+    """Like repr(), but guarantees that the result is Unicode even on Python 2.
+    """
+    return force_unicode(repr(obj), "ascii")
+
+
 def srcnameof(obj):
     """Returns the most descriptive name of a Python module, class, or function,
     including source information (filename and linenumber), if available.

src/ptvsd/common/log.py

@@ -21,6 +21,8 @@
 """Logging levels, lowest to highest importance.
 """
 
+stderr = sys.__stderr__
+
 stderr_levels = {"warning", "error"}
 """What should be logged to stderr.
 """
@@ -44,6 +46,17 @@
 _tls = threading.local()
 
 
+# Used to inject a newline into stderr if logging there, to clean up the output
+# when it's intermixed with regular prints from other sources.
+def newline(level="info"):
+    with _lock:
+        if level in stderr_levels:
+            try:
+                stderr.write("\n")
+            except Exception:
+                pass
+
+
 def write(level, text):
     assert level in LEVELS
 
@@ -62,7 +75,7 @@ def write(level, text):
     with _lock:
         if level in stderr_levels:
             try:
-                sys.__stderr__.write(output)
+                stderr.write(output)
             except Exception:
                 pass
 

src/ptvsd/common/messaging.py

@@ -70,7 +70,13 @@ def from_socket(cls, socket, name=None):
         socket.settimeout(None)  # make socket blocking
         if name is None:
             name = repr(socket)
+
+        # TODO: investigate switching to buffered sockets; readline() on unbuffered
+        # sockets is very slow! Although the implementation of readline() itself is
+        # native code, it calls read(1) in a loop - and that then ultimately calls
+        # SocketIO.readinto(), which is implemented in Python.
         socket_io = socket.makefile("rwb", 0)
+
         return cls(socket_io, socket_io, name)
 
     def __init__(self, reader, writer, name=None):
@@ -668,7 +674,7 @@ def __init__(self, stream, handlers=None, name=None):
         self.stream = stream
         self.handlers = handlers
         self.name = name if name is not None else stream.name
-        self._lock = threading.Lock()
+        self._lock = threading.RLock()
         self._stop = threading.Event()
         self._seq_iter = itertools.count(1)
         self._requests = {}

src/ptvsd/server/wrapper.py

@@ -32,7 +32,7 @@
 import _pydevd_bundle.pydevd_extension_api as pydevd_extapi  # noqa
 import _pydevd_bundle.pydevd_extension_utils as pydevd_extutil  # noqa
 import _pydevd_bundle.pydevd_frame as pydevd_frame  # noqa
-from pydevd_file_utils import get_abs_path_real_path_and_base_from_file  # noqa
+import pydevd_file_utils
 from _pydevd_bundle.pydevd_dont_trace_files import PYDEV_FILE  # noqa
 
 import ptvsd
@@ -59,8 +59,7 @@ def path_to_unicode(s):
     return s if isinstance(s, unicode) else s.decode(sys.getfilesystemencoding())
 
 
-PTVSD_DIR_PATH = os.path.dirname(os.path.abspath(get_abs_path_real_path_and_base_from_file(__file__)[0])) + os.path.sep
-NORM_PTVSD_DIR_PATH = os.path.normcase(PTVSD_DIR_PATH)
+PTVSD_DIR_PATH = pydevd_file_utils.normcase(os.path.dirname(ptvsd.__file__) + os.path.sep)
 
 
 class UnsupportedPyDevdCommandError(Exception):

tests/DEBUGGEE_PYTHONPATH/debug_me/__init__.py

@@ -40,5 +40,10 @@
 # to DebugSession - the debuggee simply needs to execute it as is.
 _code = os.getenv("PTVSD_DEBUG_ME")
 if _code:
+    # Remove it, so that subprocesses don't try to manually configure ptvsd on the
+    # same port. In multiprocess scenarios, subprocesses are supposed to load ptvsd
+    # via code that is automatically injected into the subprocess by its parent.
+    del os.environ["PTVSD_DEBUG_ME"]
+
     _code = compile(_code, "<PTVSD_DEBUG_ME>", "exec")
     eval(_code, {})

tests/DEBUGGEE_PYTHONPATH/debug_me/backchannel.py

@@ -18,45 +18,6 @@
 from ptvsd.common import fmt, log, messaging
 
 
-name = fmt("backchannel-{0}", debug_me.session_id)
-port = os.getenv("PTVSD_BACKCHANNEL_PORT")
-if port is not None:
-    port = int(port)
-    # Remove it, so that child processes don't try to use the same backchannel.
-    del os.environ["PTVSD_BACKCHANNEL_PORT"]
-
-
-if port:
-    log.info("Connecting {0} to port {1}...", name, port)
-
-    _socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    _socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-    _socket.connect(("localhost", port))
-    _stream = messaging.JsonIOStream.from_socket(_socket, name="backchannel")
-
-    @atexit.register
-    def _atexit_handler():
-        log.info("Shutting down {0}...", name)
-        try:
-            _socket.shutdown(socket.SHUT_RDWR)
-        except Exception:
-            pass
-        finally:
-            try:
-                _socket.close()
-            except Exception:
-                pass
-
-
-else:
-
-    class _stream:
-        def _error(*_):
-            raise AssertionError("Backchannel is not set up for this process")
-
-        read_json = write_json = _error
-
-
 def send(value):
     _stream.write_json(value)
 
@@ -72,3 +33,52 @@ def wait_for(expected):
         expected,
         actual,
     )
+
+
+def close():
+    global _socket, _stream
+    if _socket is None:
+        return
+
+    log.info("Shutting down {0}...", name)
+    try:
+        _socket.shutdown(socket.SHUT_RDWR)
+    except Exception:
+        pass
+    finally:
+        _socket = None
+        try:
+            _stream.close()
+        except Exception:
+            pass
+        finally:
+            _stream = None
+
+
+class _stream:
+    def _error(*_):
+        raise AssertionError("Backchannel is not set up for this process")
+
+    read_json = write_json = _error
+    close = lambda: None
+
+
+name = fmt("backchannel-{0}", debug_me.session_id)
+port = os.getenv("PTVSD_BACKCHANNEL_PORT")
+if port is not None:
+    port = int(port)
+    log.info("Connecting {0} to port {1}...", name, port)
+
+    # Remove it, so that subprocesses don't try to use the same backchannel.
+    del os.environ["PTVSD_BACKCHANNEL_PORT"]
+
+    _socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        _socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        _socket.connect(("localhost", port))
+    except Exception:
+        _socket.close()
+        raise
+    else:
+        _stream = messaging.JsonIOStream.from_socket(_socket, name="backchannel")  # noqa
+        atexit.register(close)

tests/__init__.py

@@ -11,6 +11,7 @@
 import pkgutil
 import pytest
 import py.path
+import sys
 
 # Do not import anything from ptvsd until assert rewriting is enabled below!
 
@@ -55,6 +56,7 @@ def _register_assert_rewrite(modname):
 
 # Enable full logging to stderr, and make timestamps shorter to match maximum test
 # run time better.
+log.stderr = sys.stderr  # use pytest-captured stderr rather than __stderr__
 log.stderr_levels = set(log.LEVELS)
 log.timestamp_format = "06.3f"
 

tests/code.py

@@ -12,6 +12,8 @@
 
 from ptvsd.common import compat
 
+_marked_line_numbers_cache = {}
+
 
 def get_marked_line_numbers(path):
     """Given a path to a Python source file, extracts line numbers for all lines
@@ -29,6 +31,11 @@ def get_marked_line_numbers(path):
     if isinstance(path, py.path.local):
         path = path.strpath
 
+    try:
+        return _marked_line_numbers_cache[path]
+    except KeyError:
+        pass
+
     # Read as bytes, to avoid decoding errors on Python 3.
     with open(path, "rb") as f:
         lines = {}
@@ -37,4 +44,6 @@ def get_marked_line_numbers(path):
             if match:
                 marker = compat.force_unicode(match.group(1), "ascii")
                 lines[marker] = i + 1
-        return lines
+
+    _marked_line_numbers_cache[path] = lines
+    return lines