OK Debugger: Python Tutor Tracing

client/api/assignment.py

@@ -125,6 +125,7 @@ def server_url(self):
     # hinting       -> file_contents, analytics
     # lock          -> none
     # scoring       -> none
+    # trace         -> file_contents, analytics
     # unlock        -> none
     _PROTOCOLS = [
         "file_contents",
@@ -136,6 +137,7 @@ def server_url(self):
         "lock",
         "scoring",
         "unlock",
+        "trace",
         "backup",
     ]
 

client/cli/ok.py

@@ -100,6 +100,14 @@ def parse_input(command_input=None):
     testing.add_argument('--timeout', type=int, default=10,
                         help="set the timeout duration (in seconds) for running tests")
 
+    # Debugging
+    debugging = parser.add_argument_group('debugging tools for students')
+
+    debugging.add_argument('--trace', action='store_true',
+                         help="trace code and launch python tutor")
+    debugging.add_argument('--trace-print', action='store_true',
+                         help="print the trace instead of visualizing it")
+
     # Experiments
     experiment = parser.add_argument_group('experiment options')
     experiment.add_argument('--no-experiments', action='store_true',
@@ -112,7 +120,7 @@ def parse_input(command_input=None):
                         help="launch collaborative programming environment")
 
     # Debug information
-    debug = parser.add_argument_group('debugging options')
+    debug = parser.add_argument_group('ok developer debugging options')
     debug.add_argument('--version', action='store_true',
                         help="print the version number and exit")
     debug.add_argument('--tests', action='store_true',

client/cli/publish.py

@@ -9,7 +9,7 @@
 
 OK_ROOT = os.path.normpath(os.path.dirname(client.__file__))
 CONFIG_NAME = 'config.ok'
-EXTRA_PACKAGES = ['requests']
+EXTRA_PACKAGES = ['requests', 'pytutor']
 
 def abort(message):
     print(message + ' Aborting', file=sys.stderr)

client/protocols/trace.py

@@ -0,0 +1,127 @@
+"""Implements the TraceProtocol, which traces code and
+provides a Python Tutor visualization.
+"""
+
+import datetime as dt
+import logging
+import json
+
+from pytutor import generate_trace
+from pytutor import server
+
+from client.protocols.common import models
+from client.sources.doctest import models as doctest_models
+from client.utils import format
+log = logging.getLogger(__name__)
+
+class TraceProtocol(models.Protocol):
+    """ Trace a specific test and render a JSON. """
+
+    def run(self, messages, env=None):
+        """Run gradeable tests and print results and return analytics.
+        """
+        if not self.args.trace:
+            return
+        tests = self.assignment.specified_tests
+        messages['tracing'] = {
+            'begin': get_time(),
+        }
+        if not self.args.question:
+            with format.block('*'):
+                print("Could not trace: Please specify a question to trace.")
+                print("Example: python3 ok --trace -q <name>")
+            return
+
+        test = tests[0]
+        data = test.get_code()
+        if not data:
+            with format.block('*'):
+                print("This test is not traceable.")
+            return
+
+        if isinstance(test, doctest_models.Doctest):
+            # Directly handle case (for doctests)
+            question = self.args.question[0]
+            if question not in data:
+                with format.block('*'):
+                    eligible_questions = ','.join([str(i) for i in data.keys()])
+                    print("The following doctests can be traced: {}".format(eligible_questions))
+                    usage_base = "Usage: python3 ok -q {} --trace"
+                    print(usage_base.format(eligible_questions[0]))
+                return
+            suite = [data[question]]
+        elif hasattr(test, 'suites'):
+            # Handle ok_tests
+            if not self.args.suite:
+                eligible_suite_nums = ','.join([str(i) for i in data.keys()])
+                with format.block('*'):
+                    print("Please specify a specific suite to test.")
+                    print("The following suites can be traced: {}".format(eligible_suite_nums))
+                    usage_base = "Usage: python3 ok -q {} --suite {} --trace"
+                    print(usage_base.format(self.args.question[0], eligible_suite_nums[0]))
+                return
+            if self.args.suite not in data:
+                with format.block('*'):
+                    print("Suite {} is not traceable.".format(self.args.suite))
+                return
+
+            suite = data[self.args.suite] # only trace this one suite
+            case_arg = self.args.case
+            if case_arg:
+                case_num = case_arg[0]-1
+                if not (case_arg[0]-1 not in range(len(suite))):
+                    with format.block('*'):
+                        print("You can specify a specific case to test.")
+                        print("Cases: 1-{}".format(len(suite)))
+                        usage_base = "Usage: python3 ok -q {} --suite {} --case 1 --trace"
+                        print(usage_base.format(self.args.question[0], self.args.suite))
+                        return
+                suite = [suite[case_arg[0]-1]]
+        else:
+            with format.block('*'):
+                print("This test is not traceable.")
+            return
+
+        # Setup and teardown are shared among cases within a suite.
+        setup, test_script, _ = suite_to_code(suite)
+        log.info("Starting program trace...")
+        messages['tracing']['start-trace'] = get_time()
+        modules = {k.replace('.py', '').replace('/', '.'): v for k,v in messages['file_contents'].items()}
+        data = generate_trace.run_logger(test_script, setup, modules) or "{}"
+        messages['tracing']['end-trace'] = get_time()
+        messages['tracing']['trace-len'] = len(json.loads(data).get('trace', [])) # includes the code since data is a str
+
+        if data and self.args.trace_print:
+            print(data)
+        elif data:
+            messages['tracing']['start-server'] = get_time()
+            # Open Python Tutor Browser Window with this trace
+            server.run_server(data)
+            messages['tracing']['end-server'] = get_time()
+        else:
+            print("There was an internal error while generating the trace.")
+            messages['tracing']['error'] = True
+
+
+def suite_to_code(suite):
+    code_lines = []
+    for ind, case in enumerate(suite):
+        setup = case['setup']
+        teardown = case['teardown']
+
+        case_intro = "# --- Begin Case --- #"
+        code = '\n'.join(case['code'])
+
+        # Only grab the code, since the setup/teardown is shared
+        # Render the setup as commented out lines
+        setup_code = '\n'.join(['# {}'.format(s) for s in setup.splitlines()])
+        lines = "\n{}\n{}".format(case_intro, code)
+        code_lines.append(lines)
+
+    rendered_code = setup_code + '\n' + '\n'.join(code_lines)
+    return setup, rendered_code, teardown
+
+def get_time():
+    return dt.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S:%f")
+
+protocol = TraceProtocol

client/sources/common/interpreter.py

@@ -113,6 +113,7 @@ def split_code(cls, code, PS1, PS2):
             processed_lines[-1].update(line)
         return processed_lines
 
+
     def _sync_code(self):
         """Syncs the current state of self.lines with self.code, the
         serializable string representing the set of code.
@@ -125,6 +126,41 @@ def _sync_code(self):
                 new_code.append(line)
         self.code = '\n'.join(new_code)
 
+    def _format_code_line(self, line):
+        """Remove PS1/PS2 from code lines in tests.
+        """
+        if line.startswith(self.console.PS1):
+            line = line.replace(self.console.PS1, '')
+        elif line.startswith(self.console.PS2):
+            line = line.replace(self.console.PS2, '')
+        return line
+
+    def formatted_code(self):
+        """Provides a interpretable version of the code in the case,
+        with formatting for external users (Tracing or Exporting).
+        """
+        code_lines = []
+        for line in self.lines:
+            text = line
+            if isinstance(line, CodeAnswer):
+                if line.locked:
+                    text = '# Expected: ? (test case is locked)'
+                else:
+                    split_lines = line.dump().splitlines()
+                    # Handle case when we expect multiline outputs
+                    text = '# Expected: ' + '\n# '.join(split_lines)
+            else:
+                text = self._format_code_line(line)
+            code_lines.append(text)
+        return code_lines
+
+    def formatted_setup(self):
+        return '\n'.join([self._format_code_line(l) for l in self.setup.splitlines() if l])
+
+    def formatted_teardown(self):
+        return '\n'.join([self._format_code_line(l) for l in self.teardown.splitlines() if l])
+
+
     def _construct_unique_id(self, id_prefix, lines):
         """Constructs a unique ID for a particular prompt in this case,
         based on the id_prefix and the lines in the prompt.

client/sources/doctest/models.py

@@ -19,7 +19,8 @@ class Doctest(models.Test):
     PS1 = '>>> '
     PS2 = '... '
 
-    SETUP = PS1 + 'from {} import *'
+    IMPORT_STRING = 'from {} import *'
+    SETUP = PS1 + IMPORT_STRING
     prompt_re = re.compile(r'(\s*)({}|{})'.format(PS1, '\.\.\. '))
 
     def __init__(self, file, verbose, interactive, timeout=None, **fields):
@@ -61,7 +62,7 @@ def post_instantiation(self):
                 code.append(line[len(leading_space):])
         module = self.SETUP.format(importing.path_to_module_string(self.file))
         self.case = interpreter.CodeCase(self.console, module,
-                                             code='\n'.join(code))
+                                         code='\n'.join(code))
 
     def run(self, env):
         """Runs the suites associated with this doctest.
@@ -121,3 +122,15 @@ def lock(self, hash_fn):
 
     def dump(self):
         """Doctests do not need to be dumped, since no state changes."""
+
+    def get_code(self):
+        """Render code for tracing."""
+        setup = self.IMPORT_STRING.format(importing.path_to_module_string(self.file))
+        data = {
+            self.name: {
+            'setup': setup + '\n',
+            'code': self.case.formatted_code(),
+            'teardown': '',
+            }
+        }
+        return data

client/sources/ok_test/models.py

@@ -186,6 +186,16 @@ def dump(self):
     def unique_id_prefix(self):
         return self.assignment_name + '\n' + self.name
 
+    def get_code(self):
+        extracted_code = {}
+        for ind, suite in enumerate(list(self.suites)):
+            if suite.type != 'doctest':
+                continue
+            suite_code = suite.extract_code()
+            if suite_code:
+                # Store with 1 indexed name
+                extracted_code[ind+1] = suite_code
+        return extracted_code
 
 class Suite(core.Serializable):
     type = core.String()
@@ -225,6 +235,15 @@ def enumerate_cases(self):
             return [x for x in enumerated if x[0] + 1 in self.run_only]
         return enumerated
 
+    def extract_code(self):
+        """Pull out the code for any doctest cases in the suite.
+        """
+        data = [{'setup': c.formatted_setup(),
+                 'code': c.formatted_code(),
+                 'teardown': c.formatted_teardown()} for _, c in self.enumerate_cases()
+                                                     if hasattr(c, 'setup')]
+        return data
+
     def _run_case(self, test_name, suite_number, case, case_number):
         """A wrapper for case.run().
 

demo/doctest/config.ok

@@ -1,6 +1,6 @@
 {
     "name": "Homework 1",
-    "endpoint": "https://okpy.org/path/to/endpoint",
+    "endpoint": "ok/test/su16/ex",
     "src": [
         "hw1.py"
     ],

requirements.txt

@@ -1,5 +1,6 @@
 # Client
 requests==2.12.4
+pytutor==0.1
 
 # Tests
 nose==1.3.3