[Dexter][NFC] Add Debugger Controller To Dexter

  Add DebuggerControllerBase and DefaultController to Dexter
  implements a new architecture that supports new and novel ways of running
  a debugger under dexter.
  Current implementation adds the original default behaviour via the new
  architecture via the DefaultController, this should have NFC.

Reviewers: Orlando

Differential Revision: https://reviews.llvm.org/D76926

Author: Tom Weaver

debuginfo-tests/dexter/dex/command/ParseCommand.py

@@ -13,7 +13,7 @@
 import unittest
 from copy import copy
 
-from collections import defaultdict
+from collections import defaultdict, OrderedDict
 
 from dex.utils.Exceptions import CommandParseError
 
@@ -26,7 +26,8 @@
 from dex.command.commands.DexLabel import DexLabel
 from dex.command.commands.DexUnreachable import DexUnreachable
 from dex.command.commands.DexWatch import DexWatch
-
+from dex.utils import Timer
+from dex.utils.Exceptions import CommandParseError, DebuggerException
 
 def _get_valid_commands():
     """Return all top level DExTer test commands.
@@ -262,9 +263,7 @@ def _find_all_commands_in_file(path, file_lines, valid_commands):
         raise format_parse_err(msg, path, file_lines, err_point)
     return dict(commands)
 
-
-
-def find_all_commands(source_files):
+def _find_all_commands(source_files):
     commands = defaultdict(dict)
     valid_commands = _get_valid_commands()
     for source_file in source_files:
@@ -277,6 +276,21 @@ def find_all_commands(source_files):
 
     return dict(commands)
 
+def get_command_infos(source_files):
+  with Timer('parsing commands'):
+      try:
+          commands = _find_all_commands(source_files)
+          command_infos = OrderedDict()
+          for command_type in commands:
+              for command in commands[command_type].values():
+                  if command_type not in command_infos:
+                      command_infos[command_type] = []
+                  command_infos[command_type].append(command)
+          return OrderedDict(command_infos)
+      except CommandParseError as e:
+          msg = 'parser error: <d>{}({}):</> {}\n{}\n{}\n'.format(
+                e.filename, e.lineno, e.info, e.src, e.caret)
+          raise DebuggerException(msg)
 
 class TestParseCommand(unittest.TestCase):
     class MockCmd(CommandBase):

debuginfo-tests/dexter/dex/command/__init__.py

@@ -5,5 +5,5 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-from dex.command.ParseCommand import find_all_commands
+from dex.command.ParseCommand import get_command_infos
 from dex.command.StepValueInfo import StepValueInfo

debuginfo-tests/dexter/dex/command/commands/DexExpectStepOrder.py

@@ -6,6 +6,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from dex.command.CommandBase import CommandBase
+from dex.dextIR import LocIR
 from dex.dextIR import ValueIR
 
 class DexExpectStepOrder(CommandBase):
@@ -28,11 +29,9 @@ def __init__(self, *args):
     def get_name():
         return __class__.__name__
 
-    def eval(self, debugger):
-        step_info = debugger.get_step_info()
-        loc = step_info.current_location
-        return {'DexExpectStepOrder': ValueIR(expression=str(loc.lineno),
-                      value=str(debugger.step_index), type_name=None,
+    def eval(self, step_info):
+        return {'DexExpectStepOrder': ValueIR(expression=str(step_info.current_location.lineno),
+                      value=str(step_info.step_index), type_name=None,
                       error_string=None,
                       could_evaluate=True,
                       is_optimized_away=True,

debuginfo-tests/dexter/dex/command/commands/DexUnreachable.py

@@ -26,7 +26,7 @@ def __init(self):
     def get_name():
         return __class__.__name__
 
-    def eval(self, debugger):
+    def eval(self, step_info):
         # If we're ever called, at all, then we're evaluating a line that has
         # been marked as unreachable. Which means a failure.
         vir = ValueIR(expression="Unreachable",

debuginfo-tests/dexter/dex/debugger/DebuggerBase.py

@@ -7,10 +7,7 @@
 """Base class for all debugger interface implementations."""
 
 import abc
-from itertools import chain
-import os
 import sys
-import time
 import traceback
 
 from dex.dextIR import DebuggerIR, ValueIR
@@ -20,28 +17,22 @@
 
 
 class DebuggerBase(object, metaclass=abc.ABCMeta):
-    def __init__(self, context, step_collection):
+    def __init__(self, context):
         self.context = context
-        self.steps = step_collection
         self._interface = None
         self.has_loaded = False
         self._loading_error = NotYetLoadedDebuggerException()
-        self.watches = set()
-
         try:
             self._interface = self._load_interface()
             self.has_loaded = True
             self._loading_error = None
         except DebuggerException:
             self._loading_error = sys.exc_info()
 
-        self.step_index = 0
-
     def __enter__(self):
         try:
             self._custom_init()
             self.clear_breakpoints()
-            self.add_breakpoints()
         except DebuggerException:
             self._loading_error = sys.exc_info()
         return self
@@ -86,80 +77,13 @@ def loading_error_trace(self):
         tb = ''.join(tb).splitlines(True)
         return tb
 
-    def add_breakpoints(self):
-        for s in self.context.options.source_files:
-            with open(s, 'r') as fp:
-                num_lines = len(fp.readlines())
-            for line in range(1, num_lines + 1):
-                self.add_breakpoint(s, line)
-
-    def _update_step_watches(self, step_info):
-        loc = step_info.current_location
-        watch_cmds = ['DexUnreachable', 'DexExpectStepOrder']
-        towatch = chain.from_iterable(self.steps.commands[x]
-                                      for x in watch_cmds
-                                      if x in self.steps.commands)
-        try:
-            # Iterate over all watches of the types named in watch_cmds
-            for watch in towatch:
-                if (os.path.exists(loc.path)
-                        and os.path.samefile(watch.path, loc.path)
-                        and watch.lineno == loc.lineno):
-                    result = watch.eval(self)
-                    step_info.watches.update(result)
-                    break
-        except KeyError:
-            pass
-
     def _sanitize_function_name(self, name):  # pylint: disable=no-self-use
         """If the function name returned by the debugger needs any post-
         processing to make it fit (for example, if it includes a byte offset),
         do that here.
         """
         return name
 
-    def start(self):
-        self.steps.clear_steps()
-        self.launch()
-
-        for command_obj in chain.from_iterable(self.steps.commands.values()):
-            self.watches.update(command_obj.get_watches())
-
-        max_steps = self.context.options.max_steps
-        for _ in range(max_steps):
-            while self.is_running:
-                pass
-
-            if self.is_finished:
-                break
-
-            self.step_index += 1
-            step_info = self.get_step_info()
-
-            if step_info.current_frame:
-                self._update_step_watches(step_info)
-                self.steps.new_step(self.context, step_info)
-
-            if self.in_source_file(step_info):
-                self.step()
-            else:
-                self.go()
-
-            time.sleep(self.context.options.pause_between_steps)
-        else:
-            raise DebuggerException(
-                'maximum number of steps reached ({})'.format(max_steps))
-
-    def in_source_file(self, step_info):
-        if not step_info.current_frame:
-            return False
-        if not step_info.current_location.path:
-            return False
-        if not os.path.exists(step_info.current_location.path):
-            return False
-        return any(os.path.samefile(step_info.current_location.path, f) \
-                   for f in self.context.options.source_files)
-
     @abc.abstractmethod
     def _load_interface(self):
         pass
@@ -209,7 +133,7 @@ def go(self) -> ReturnCode:
         pass
 
     @abc.abstractmethod
-    def get_step_info(self):
+    def get_step_info(self, watches, step_index):
         pass
 
     @abc.abstractproperty

debuginfo-tests/dexter/dex/debugger/DebuggerControllers/DebuggerControllerBase.py

@@ -0,0 +1,27 @@
+# DExTer : Debugging Experience Tester
+# ~~~~~~   ~         ~~         ~   ~~
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Default class for controlling debuggers."""
+
+import abc
+
+class DebuggerControllerBase(object, metaclass=abc.ABCMeta):
+    @abc.abstractclassmethod
+    def _run_debugger_custom(self):
+        """Specify your own implementation of run_debugger_custom in your own
+        controller.
+        """
+        pass
+
+    def run_debugger(self, debugger):
+        """Responsible for correctly launching and tearing down the debugger.
+        """
+        self.debugger = debugger
+        with self.debugger:
+            self._run_debugger_custom()
+        # We may need to pickle this debugger controller after running the
+        # debugger. Debuggers are not picklable objects, so set to None.
+        self.debugger = None

debuginfo-tests/dexter/dex/debugger/DebuggerControllers/DefaultController.py

@@ -0,0 +1,90 @@
+# DExTer : Debugging Experience Tester
+# ~~~~~~   ~         ~~         ~   ~~
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Base class for controlling debuggers."""
+
+from itertools import chain
+import os
+import time
+
+from dex.debugger.DebuggerControllers.DebuggerControllerBase import DebuggerControllerBase
+from dex.utils.Exceptions import DebuggerException
+
+class DefaultController(DebuggerControllerBase):
+    def __init__(self, context, step_collection):
+        self.context = context
+        self.step_collection = step_collection
+        self.watches = set()
+        self.step_index = 0
+
+    def _update_step_watches(self, step_info):
+        watch_cmds = ['DexUnreachable', 'DexExpectStepOrder']
+        towatch = chain.from_iterable(self.step_collection.commands[x]
+                                      for x in watch_cmds
+                                      if x in self.step_collection.commands)
+        try:
+            # Iterate over all watches of the types named in watch_cmds
+            for watch in towatch:
+                loc = step_info.current_location
+                if (os.path.exists(loc.path)
+                        and os.path.samefile(watch.path, loc.path)
+                        and watch.lineno == loc.lineno):
+                    result = watch.eval(step_info)
+                    step_info.watches.update(result)
+                    break
+        except KeyError:
+            pass
+
+    def _break_point_all_lines(self):
+        for s in self.context.options.source_files:
+            with open(s, 'r') as fp:
+                num_lines = len(fp.readlines())
+            for line in range(1, num_lines + 1):
+                self.debugger.add_breakpoint(s, line)
+
+    def _in_source_file(self, step_info):
+        if not step_info.current_frame:
+            return False
+        if not step_info.current_location.path:
+            return False
+        if not os.path.exists(step_info.current_location.path):
+            return False
+        return any(os.path.samefile(step_info.current_location.path, f) \
+                   for f in self.context.options.source_files)
+
+    def _run_debugger_custom(self):
+        self.step_collection.debugger = self.debugger.debugger_info
+        self._break_point_all_lines()
+
+        self.debugger.launch()
+
+        for command_obj in chain.from_iterable(self.step_collection.commands.values()):
+            self.watches.update(command_obj.get_watches())
+
+        max_steps = self.context.options.max_steps
+        for _ in range(max_steps):
+            while self.debugger.is_running:
+                pass
+
+            if self.debugger.is_finished:
+                break
+
+            self.step_index += 1
+            step_info = self.debugger.get_step_info(self.watches, self.step_index)
+
+            if step_info.current_frame:
+                self._update_step_watches(step_info)
+                self.step_collection.new_step(self.context, step_info)
+
+            if self._in_source_file(step_info):
+                self.debugger.step()
+            else:
+                self.debugger.go()
+
+            time.sleep(self.context.options.pause_between_steps)
+        else:
+            raise DebuggerException(
+                'maximum number of steps reached ({})'.format(max_steps))