Basics of tracing

client/api/assignment.py

@@ -123,6 +123,7 @@ def server_url(self):
     # file_contents -> none
     # grading       -> none
     # hinting       -> file_contents, analytics
+    # trace         -> file_contents, analytics
     # lock          -> none
     # scoring       -> none
     # unlock        -> none
@@ -136,6 +137,7 @@ def server_url(self):
         "lock",
         "scoring",
         "unlock",
+        "trace",
         "backup",
     ]
 

client/cli/ok.py

@@ -99,6 +99,8 @@ def parse_input(command_input=None):
                         help="submit composition revision")
     testing.add_argument('--timeout', type=int, default=10,
                         help="set the timeout duration (in seconds) for running tests")
+    testing.add_argument('--trace', action='store_true',
+                         help="trace code and launch python tutor")
 
     # Experiments
     experiment = parser.add_argument_group('experiment options')

client/protocols/trace.py

@@ -0,0 +1,99 @@
+"""Implements the TraceProtocol, which traces code and
+provides a Python Tutor visualization.
+"""
+
+import logging
+
+from client.protocols.common import models
+from client.sources.doctest import models as doctest_models
+
+from client.utils import format
+from client.pythontutor import generate_trace
+log = logging.getLogger(__name__)
+
+class TraceProtocol(models.Protocol):
+    """ Trace a specific test and render a JSON. """
+
+    def run(self, messages, env=None):
+        """Run gradeable tests and print results and return analytics.
+        """
+        if not self.args.trace:
+            return
+        tests = self.assignment.specified_tests
+        if not self.args.question:
+            with format.block('*'):
+                print("Could not trace: Please specify a question to trace.")
+                print("Example: python3 ok --trace -q <name>")
+            return
+
+        test = tests[0]
+        # TODO: Check for appropriate instance types
+        data = test.get_code()
+        if not data:
+            with format.block('*'):
+                print("This test is not traceable.")
+            return
+
+        if isinstance(test, doctest_models.Doctest):
+            # Directly handle case (for doctests)
+            question = self.args.question[0]
+            if question not in data:
+                with format.block('*'):
+                    eligible_questions = ','.join([str(i) for i in data.keys()])
+                    print("The following doctests can be traced: {}".format(eligible_questions))
+                    usage_base = "Usage: python3 ok -q {} --trace"
+                    usage_base.format(eligible_questions[0])
+                    return
+            suite = [data[question]]
+        elif hasattr(test, 'suites'):
+            # Handle ok_tests
+            if self.args.suite:
+                eligible_suite_nums = ','.join([str(i) for i in data.keys()])
+                with format.block('*'):
+                    print("Please specify a specific suite to test.")
+                    print("The following suites can be traced: {}".format(eligible_suite_nums))
+                    usage_base = "Usage: python3 ok -q {} --suite {} --trace"
+                    usage_base.format(self.args.question[0], eligible_suite_nums[0])
+                return
+            if self.args.suite not in data:
+                with format.block('*'):
+                    print("Suite {} is not traceable.".format(self.args.suite))
+                return
+
+            suite = data[self.args.suite] # only trace this one suite
+            case_arg = self.args.case
+            if case_arg:
+                case_num = case_arg[0]-1
+                if not (case_arg[0]-1 not in range(len(suite))):
+                    with format.block('*'):
+                        print("You can specify a specific case to test.")
+                        print("Cases: 1-{}".format(len(suite)))
+                        usage_base = "Usage: python3 ok -q {} --suite {} --case 1 --trace"
+                        print(usage_base.format(self.args.question[0], self.args.suite))
+                        return
+                suite = [suite[case_arg[0]-1]]
+        else:
+            pass
+
+        # Setup and teardown are shared among cases within a suite.
+        setup, test_script, _ = suite_to_code(suite)
+        data = generate_trace.run_logger(test_script, setup, {})
+        print(data)
+
+        # Call Python Tutor with the approriate values
+
+def suite_to_code(suite):
+    code_lines = []
+    for ind, case in enumerate(suite):
+        setup = case['setup']
+        teardown = case['teardown']
+
+        case_intro = "# --- Begin Case --- #"
+        code = '\n'.join(case['code'])
+
+        # Only grab the code, since the setup/teardown is shared
+        lines = "{}\n{}".format(case_intro, code)
+        code_lines.append(lines)
+    return setup, '\n'.join(code_lines), teardown
+
+protocol = TraceProtocol

client/sources/common/interpreter.py

@@ -113,6 +113,7 @@ def split_code(cls, code, PS1, PS2):
             processed_lines[-1].update(line)
         return processed_lines
 
+
     def _sync_code(self):
         """Syncs the current state of self.lines with self.code, the
         serializable string representing the set of code.
@@ -125,6 +126,41 @@ def _sync_code(self):
                 new_code.append(line)
         self.code = '\n'.join(new_code)
 
+    def _format_code_line(self, line):
+        """Remove PS1/PS2 from code lines in tests.
+        """
+        if line.startswith(self.console.PS1):
+            line = line.replace(self.console.PS1, '')
+        elif line.startswith(self.console.PS2):
+            line = line.replace(self.console.PS2, '')
+        return line
+
+    def formatted_code(self):
+        """Provides a interpretable version of the code in the case,
+        with formatting for external users (Tracing or Exporting).
+        """
+        code_lines = []
+        for line in self.lines:
+            text = line
+            if isinstance(line, CodeAnswer):
+                if line.locked:
+                    text = '# Expected: ? (test case is locked)'
+                else:
+                    split_lines = line.dump().splitlines()
+                    # Handle case when we expect multiline outputs
+                    text = '# Expected: ' + '\n# '.join(split_lines)
+            else:
+                text = self._format_code_line(line)
+            code_lines.append(text)
+        return code_lines
+
+    def formatted_setup(self):
+        return '\n'.join([self._format_code_line(l) for l in self.setup.splitlines() if l])
+
+    def formatted_teardown(self):
+        return '\n'.join([self._format_code_line(l) for l in self.teardown.splitlines() if l])
+
+
     def _construct_unique_id(self, id_prefix, lines):
         """Constructs a unique ID for a particular prompt in this case,
         based on the id_prefix and the lines in the prompt.

client/sources/doctest/models.py

@@ -19,7 +19,8 @@ class Doctest(models.Test):
     PS1 = '>>> '
     PS2 = '... '
 
-    SETUP = PS1 + 'from {} import *'
+    IMPORT_STRING = 'from {} import *'
+    SETUP = PS1 + IMPORT_STRING
     prompt_re = re.compile(r'(\s*)({}|{})'.format(PS1, '\.\.\. '))
 
     def __init__(self, file, verbose, interactive, timeout=None, **fields):
@@ -61,7 +62,7 @@ def post_instantiation(self):
                 code.append(line[len(leading_space):])
         module = self.SETUP.format(importing.path_to_module_string(self.file))
         self.case = interpreter.CodeCase(self.console, module,
-                                             code='\n'.join(code))
+                                         code='\n'.join(code))
 
     def run(self, env):
         """Runs the suites associated with this doctest.
@@ -121,3 +122,15 @@ def lock(self, hash_fn):
 
     def dump(self):
         """Doctests do not need to be dumped, since no state changes."""
+
+    def get_code(self):
+        """Render code for tracing."""
+        setup = self.IMPORT_STRING.format(importing.path_to_module_string(self.file))
+        data = {
+            self.name: {
+            'setup': setup + '\n',
+            'code': self.case.formatted_code(),
+            'teardown': '',
+            }
+        }
+        return data

client/sources/ok_test/models.py

@@ -186,6 +186,16 @@ def dump(self):
     def unique_id_prefix(self):
         return self.assignment_name + '\n' + self.name
 
+    def get_code(self):
+        extracted_code = {}
+        for ind, suite in enumerate(list(self.suites)):
+            if suite.type != 'doctest':
+                continue
+            suite_code = suite.extract_code()
+            if suite_code:
+                # Store with 1 indexed name
+                extracted_code[ind+1] = suite_code
+        return extracted_code
 
 class Suite(core.Serializable):
     type = core.String()
@@ -225,6 +235,15 @@ def enumerate_cases(self):
             return [x for x in enumerated if x[0] + 1 in self.run_only]
         return enumerated
 
+    def extract_code(self):
+        """Pull out the code for any doctest cases in the suite.
+        """
+        data = [{'setup': c.formatted_setup(),
+                 'code': c.formatted_code(),
+                 'teardown': c.formatted_teardown()} for _, c in self.enumerate_cases()
+                                                     if hasattr(c, 'setup')]
+        return data
+
     def _run_case(self, test_name, suite_number, case, case_number):
         """A wrapper for case.run().