Django test fixes.

Fix test logs not being captured by pytest.

Fix "import debug_me" check improperly applied in tests where it is unnecessary.

Fix some clarifying patterns not respecting the underlying pattern.

Add pattern helpers for strings: starting_with, ending_with, containing.

Move DAP test helpers to a separate module, and add a helper for frames.

Various Unicode handling test fixes.

src/ptvsd/common/compat.py

@@ -71,7 +71,7 @@ def force_bytes(s, encoding, errors="strict"):
         return s
 
 
-def force_str(s, encoding, errors="strict"):
+def force_str(s, encoding="ascii", errors="strict"):
     """Converts s to str (which is bytes on Python 2, and unicode on Python 3), using
     the provided encoding if necessary. If s is already str, it is returned as is.
 
@@ -137,6 +137,12 @@ def nameof(obj, quote=False):
     return force_unicode(name, "utf-8", "replace")
 
 
+def unicode_repr(obj):
+    """Like repr(), but guarantees that the result is Unicode even on Python 2.
+    """
+    return force_unicode(repr(obj), "ascii")
+
+
 def srcnameof(obj):
     """Returns the most descriptive name of a Python module, class, or function,
     including source information (filename and linenumber), if available.

src/ptvsd/common/log.py

@@ -21,6 +21,8 @@
 """Logging levels, lowest to highest importance.
 """
 
+stderr = sys.__stderr__
+
 stderr_levels = {"warning", "error"}
 """What should be logged to stderr.
 """
@@ -62,7 +64,7 @@ def write(level, text):
     with _lock:
         if level in stderr_levels:
             try:
-                sys.__stderr__.write(output)
+                stderr.write(output)
             except Exception:
                 pass
 

src/ptvsd/common/messaging.py

@@ -70,7 +70,13 @@ def from_socket(cls, socket, name=None):
         socket.settimeout(None)  # make socket blocking
         if name is None:
             name = repr(socket)
+
+        # TODO: investigate switching to buffered sockets; readline() on unbuffered
+        # sockets is very slow! Although the implementation of readline() itself is
+        # native code, it calls read(1) in a loop - and that then ultimately calls
+        # SocketIO.readinto(), which is implemented in Python.
         socket_io = socket.makefile("rwb", 0)
+
         return cls(socket_io, socket_io, name)
 
     def __init__(self, reader, writer, name=None):

tests/DEBUGGEE_PYTHONPATH/debug_me/__init__.py

@@ -40,5 +40,10 @@
 # to DebugSession - the debuggee simply needs to execute it as is.
 _code = os.getenv("PTVSD_DEBUG_ME")
 if _code:
+    # Remove it, so that subprocesses don't try to manually configure ptvsd on the
+    # same port. In multiprocess scenarios, subprocesses are supposed to load ptvsd
+    # via code that is automatically injected into the subprocess by its parent.
+    del os.environ["PTVSD_DEBUG_ME"]
+
     _code = compile(_code, "<PTVSD_DEBUG_ME>", "exec")
     eval(_code, {})

tests/DEBUGGEE_PYTHONPATH/debug_me/backchannel.py

@@ -22,7 +22,7 @@
 port = os.getenv("PTVSD_BACKCHANNEL_PORT")
 if port is not None:
     port = int(port)
-    # Remove it, so that child processes don't try to use the same backchannel.
+    # Remove it, so that subprocesses don't try to use the same backchannel.
     del os.environ["PTVSD_BACKCHANNEL_PORT"]
 
 

tests/__init__.py

@@ -11,6 +11,7 @@
 import pkgutil
 import pytest
 import py.path
+import sys
 
 # Do not import anything from ptvsd until assert rewriting is enabled below!
 
@@ -55,6 +56,7 @@ def _register_assert_rewrite(modname):
 
 # Enable full logging to stderr, and make timestamps shorter to match maximum test
 # run time better.
+log.stderr = sys.stderr  # use pytest-captured stderr rather than __stderr__
 log.stderr_levels = set(log.LEVELS)
 log.timestamp_format = "06.3f"
 

tests/debug.py

@@ -4,7 +4,7 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-from collections import namedtuple
+import collections
 import itertools
 import os
 import platform
@@ -22,7 +22,7 @@
 import tests
 from tests import net
 from tests.patterns import some
-from tests.timeline import Timeline, Event, Response
+from tests.timeline import Timeline, Event, Request, Response
 
 PTVSD_DIR = py.path.local(ptvsd.__file__) / ".."
 PTVSD_PORT = net.get_test_server_port(5678, 5800)
@@ -41,7 +41,7 @@
 """
 
 
-StopInfo = namedtuple('StopInfo', [
+StopInfo = collections.namedtuple('StopInfo', [
     'body',
     'frames',
     'thread_id',
@@ -51,11 +51,24 @@
 
 class Session(object):
     WAIT_FOR_EXIT_TIMEOUT = 10
+    """Timeout used by wait_for_exit() before it kills the ptvsd process.
+    """
+
+    START_METHODS = {
+        'launch',  # ptvsd --client ... foo.py
+        'attach_socket_cmdline',  #  ptvsd ... foo.py
+        'attach_socket_import',  #  python foo.py (foo.py must import debug_me)
+        'attach_pid',  # python foo.py && ptvsd ... --pid
+        'custom_client'  # python foo.py (foo.py has to manually connect to session)
+    }
+
+    DEBUG_ME_START_METHODS = {"attach_socket_import"}
+    """Start methods that require import debug_me."""
 
     _counter = itertools.count(1)
 
     def __init__(self, start_method='launch', ptvsd_port=None, pid=None):
-        assert start_method in ('launch', 'attach_pid', 'attach_socket_cmdline', 'attach_socket_import', 'custom_client')
+        assert start_method in self.START_METHODS
         assert ptvsd_port is None or start_method.startswith('attach_socket_')
 
         self.id = next(self._counter)
@@ -237,11 +250,13 @@ def _validate_pyfile(self, filename):
         assert os.path.isfile(filename)
         with open(filename, "rb") as f:
             code = f.read()
-            if self.start_method != "custom_client":
-                assert b"debug_me" in code, (
-                    "Python source code that is run via tests.debug.Session must "
-                    "import debug_me"
+            if self.start_method in self.DEBUG_ME_START_METHODS:
+                assert b"debug_me" in code, fmt(
+                    "{0} is started via {1}, but it doesn't import debug_me.",
+                    filename,
+                    self.start_method,
                 )
+
             return code
 
     def _get_target(self):
@@ -289,7 +304,7 @@ def _setup_session(self, **kwargs):
         for k, v in kwargs.items():
             setattr(self, k, v)
 
-        assert self.start_method in ('launch', 'attach_pid', 'attach_socket_cmdline', 'attach_socket_import', 'custom_client')
+        assert self.start_method in self.START_METHODS
         assert len(self.target) == 2
         assert self.target[0] in ('file', 'module', 'code')
 
@@ -371,7 +386,7 @@ def initialize(self, **kwargs):
         # Assume that values are filenames - it's usually either that, or numbers.
         make_filename = compat.filename_bytes if sys.version_info < (3,) else compat.filename
         env = {
-            compat.force_str(k, "ascii"): make_filename(v)
+            compat.force_str(k): make_filename(v)
             for k, v in self.env.items()
         }
 
@@ -695,18 +710,26 @@ def _process_response(self, request_occ, response):
     def _process_request(self, request):
         assert False, 'ptvsd should not be sending requests.'
 
-    def request_continue(self):
-        self.send_request('continue').wait_for_response(freeze=False)
-
-    def set_breakpoints(self, path, lines=()):
-        return self.request('setBreakpoints', arguments={
-            'source': {'path': path},
-            'breakpoints': [{'line': bp_line} for bp_line in lines],
-        }).get('breakpoints', {})
-
     def wait_for_next_event(self, event, body=some.object):
         return self.timeline.wait_for_next(Event(event, body)).body
 
+    def output(self, category):
+        """Returns all output of a given category as a single string, assembled from
+        all the "output" events received for that category so far.
+        """
+        events = self.all_occurrences_of(
+            Event("output", some.dict.containing({"category": category}))
+        )
+        return "".join(event.body["output"] for event in events)
+
+    def captured_stdout(self, encoding=None):
+        return self.captured_output.stdout(encoding)
+
+    def captured_stderr(self, encoding=None):
+        return self.captured_output.stderr(encoding)
+
+    # Helpers for specific DAP patterns.
+
     def wait_for_stop(self, reason=some.str, expected_frames=None, expected_text=None, expected_description=None):
         stopped_event = self.wait_for_next(Event('stopped', some.dict.containing({'reason': reason})))
         stopped = stopped_event.body
@@ -737,7 +760,66 @@ def wait_for_stop(self, reason=some.str, expected_frames=None, expected_text=Non
 
         return StopInfo(stopped, frames, tid, fid)
 
-    def connect_to_child_session(self, ptvsd_subprocess):
+    def request_continue(self):
+        self.send_request('continue').wait_for_response(freeze=False)
+
+    def set_breakpoints(self, path, lines=()):
+        return self.request('setBreakpoints', arguments={
+            'source': {'path': path},
+            'breakpoints': [{'line': bp_line} for bp_line in lines],
+        }).get('breakpoints', {})
+
+    def get_variables(self, *varnames, **kwargs):
+        """Fetches the specified variables from the frame specified by frame_id, or
+        from the topmost frame in the last "stackTrace" response if frame_id is not
+        specified.
+
+        If varnames is empty, then all variables in the frame are returned. The result
+        is an OrderedDict, in which every entry has variable name as the key, and a
+        DAP Variable object as the value. The original order of variables as reported
+        by the debugger is preserved.
+
+        If varnames is not empty, then only the specified variables are returned.
+        The result is a tuple, in which every entry is a DAP Variable object; those
+        entries are in the same order as varnames.
+        """
+
+        assert self.timeline.is_frozen
+
+        frame_id = kwargs.pop("frame_id", None)
+        if frame_id is None:
+            stackTrace_responses = self.all_occurrences_of(
+                Response(Request("stackTrace"))
+            )
+            assert stackTrace_responses, (
+                'get_variables() without frame_id requires at least one response '
+                'to a "stackTrace" request in the timeline.'
+            )
+            stack_trace = stackTrace_responses[-1].body
+            frame_id = stack_trace["stackFrames"][0]["id"]
+
+        scopes = self.request("scopes", {"frameId": frame_id})["scopes"]
+        assert len(scopes) > 0
+
+        variables = self.request(
+            "variables", {"variablesReference": scopes[0]["variablesReference"]}
+        )["variables"]
+
+        variables = collections.OrderedDict(((v["name"], v) for v in variables))
+        if varnames:
+            assert set(varnames) <= set(variables.keys())
+            return tuple((variables[name] for name in varnames))
+        else:
+            return variables
+
+    def get_variable(self, varname, frame_id=None):
+        """Same as get_variables(...)[0].
+        """
+        return self.get_variables(varname, frame_id=frame_id)[0]
+
+    def attach_to_subprocess(self, ptvsd_subprocess):
+        assert ptvsd_subprocess == Event("ptvsd_subprocess")
+
         child_port = ptvsd_subprocess.body['port']
         assert child_port != 0
 
@@ -754,11 +836,20 @@ def connect_to_child_session(self, ptvsd_subprocess):
         else:
             return child_session
 
-    def connect_to_next_child_session(self):
+    def attach_to_next_subprocess(self):
         ptvsd_subprocess = self.wait_for_next(Event('ptvsd_subprocess'))
-        return self.connect_to_child_session(ptvsd_subprocess)
+        return self.attach_to_subprocess(ptvsd_subprocess)
+
+    def reattach(self, **kwargs):
+        """Creates and initializes a new Session that tries to attach to the same
+        process.
+
+        Upon return, handshake() has been performed, but the caller is responsible
+        for invoking start_debugging().
+        """
+
+        assert self.start_method.startswith("attach_socket_")
 
-    def connect_with_new_session(self, **kwargs):
         ns = Session(start_method='attach_socket_import', ptvsd_port=self.ptvsd_port)
         try:
             ns._setup_session(**kwargs)
@@ -776,24 +867,10 @@ def connect_with_new_session(self, **kwargs):
             ns.handshake()
         except Exception:
             ns.close()
+            raise
         else:
             return ns
 
-    def output(self, category):
-        """Returns all output of a given category as a single string, assembled from
-        all the "output" events received for that category so far.
-        """
-        events = self.all_occurrences_of(
-            Event("output", some.dict.containing({"category": category}))
-        )
-        return "".join(event.body["output"] for event in events)
-
-    def captured_stdout(self, encoding=None):
-        return self.captured_output.stdout(encoding)
-
-    def captured_stderr(self, encoding=None):
-        return self.captured_output.stderr(encoding)
-
 
 class CapturedOutput(object):
     """Captured stdout and stderr of the debugged process.