byterun: minor changes for debugging.

- Fix recursive GuestException structure.  (Note: GuestException is only
  enabled if VirtualMachine.more_info is turned on.)

- Copied a bug chunk of the stdlib's sre_parse.py into
  opy/testdata/regex_compile.py so we can tickle a bug.  I want to be
  able to modify the source, and imports are somewhat broken anyway
  (host and guest confusion.)

- Add VirtualMachine.repr_ok, which can be turned off to not call repr()
  for our debug prints.  For some reason that interacts with
    SubPattern.__repr__ in sre_parse.py.
  - Able to run testdata/repr_method.py with byterun now.

- Remove dependency on stdlib logging module.

- A bunch of messy debug prints.  Trying to track down the
  regex_compile.py bug.

opy/byterun/pyobj.py

@@ -8,6 +8,12 @@
import types


def debug1(msg, *args):
  if args:
    msg = msg % args
  print(msg, file=sys.stderr)


def make_cell(value):
    # Thanks to Alex Gaynor for help with this bit of twistiness.
    # Construct an actual cell object by creating a closure right here,
@@ -17,6 +23,13 @@ def make_cell(value):


class Function(object):
    """
    CPython equivalent:

    x = PyFunction_New(v, f->f_globals);
    PyFunction_SetClosure(x, v)
    """

    __slots__ = [
        'func_code', 'func_name', 'func_defaults', 'func_globals',
        'func_locals', 'func_dict', 'func_closure',
@@ -46,14 +59,14 @@ def __init__(self, name, code, globs, locs, defaults, closure, vm):
        self._func = types.FunctionType(code, globs, **kw)

    def __repr__(self):         # pragma: no cover
        return '<byterun Function %s at 0x%08x>' % (
            self.func_name, id(self)
        )
        return '<byterun Function %s at 0x%08x>' % (self.func_name, id(self))

    def __get__(self, instance, owner):
        if instance is not None:
            return Method(instance, owner, self)
        return Method(None, owner, self)
        # used in unit tests?  But I don't see it triggered in real code.
        #raise AssertionError('Function.__get__')
        m = Method(instance, owner, self)
        #debug1('*** METHOD %s', m)
        return m

    def __call__(self, *args, **kwargs):
        #if PY2 and self.func_name in ["<setcomp>", "<dictcomp>", "<genexpr>"]:
@@ -85,9 +98,11 @@ def __call__(self, *args, **kwargs):
            frame.generator = gen
            retval = gen
        else:
            # NOTE: Can raise exceptions!
            retval = self._vm.run_frame(frame)
        return retval


class Method(object):
    def __init__(self, obj, _class, func):
        self.im_self = obj
@@ -347,6 +362,8 @@ def line_number(self):


class Generator(object):
    """A wrapper around a Frame that can run for one generator tick."""

    def __init__(self, g_frame, vm):
        self.g_frame = g_frame
        self._vm = vm
@@ -373,6 +390,7 @@ def send(self, value=None):
            raise TypeError("Can't send non-None value to a just-started generator")
        self.g_frame.stack.append(value)
        self.started = True
        # NOTE: Can raise exceptions!
        val = self._vm.resume_frame(self.g_frame)
        if self.finished:
            raise StopIteration(val)

opy/byterun/pyvm2.py

@@ -6,20 +6,18 @@
import dis
import inspect
import linecache
import logging
import operator
import repr
import repr as repr_lib  # Don't conflict with builtin repr()
import sys
import traceback
import types

# Function used in MAKE_FUNCTION, MAKE_CLOSURE
# Generator used in YIELD_FROM, which we might not need.
from pyobj import Frame, Block, Function, Generator

log = logging.getLogger(__name__)

# Create a repr that won't overflow.
repr_obj = repr.Repr()
repr_obj = repr_lib.Repr()
repr_obj.maxother = 120
repper = repr_obj.repr

@@ -54,6 +52,8 @@ class GuestException(Exception):
  
    def __init__(self, exctype, value, frames):
        self.exctype = exctype
        if isinstance(value, GuestException):
          raise AssertionError
        self.value = value
        self.frames = frames

@@ -65,7 +65,7 @@ def __str__(self):
            filename = f.f_code.co_filename
            lineno = f.line_number()
            parts.append(
                '  File "%s", line %d, in %s' %
                '- File "%s", line %d, in %s' %
                (filename, lineno, f.f_code.co_name))
            linecache.checkcache(filename)
            line = linecache.getline(filename, lineno, f.f_globals)
@@ -102,6 +102,10 @@ def __init__(self, subset=False, verbose=VERBOSE):
        self.more_info = False
        #self.more_info = True
        self.verbose = verbose
        # some objects define __repr__, which means our debug() logging screws
        # things up!  Even though they don't have side effects, this somehow
        # matters.
        self.repr_ok = True

        # The call stack of frames.
        self.frames = []
@@ -135,7 +139,8 @@ def make_frame(self, code, callargs={}, f_globals=None, f_locals=None):
        """
        Called by self.run_code and Function.__call__.
        """
        log.info("make_frame: code=%r, callargs=%s" % (code, repper(callargs)))
        # NOTE: repper causes problems running code!  See testdata/repr_method.py
        #debug("make_frame: code=%r, callargs=%s", code, repper(callargs))
        if f_globals is not None:
            f_globals = f_globals
            if f_locals is None:
@@ -165,22 +170,27 @@ def logTick(self, byteName, arguments, opoffset, linestarts):
        """ Log arguments, block stack, and data stack for each opcode."""
        indent = "    " * (len(self.frames)-1)
        stack_rep = repper(self.frame.stack)
        block_stack_rep = repper(self.frame.block_stack)

        if arguments:
        #block_stack_rep = repper(self.frame.block_stack)
        # repr_lib is causing problems
        if self.repr_ok:
            stack_rep = repr(self.frame.stack)
            #block_stack_rep = repr(self.frame.block_stack)

        arg_str = ''
        if arguments and self.repr_ok:
            arg_str = ' %r' % (arguments[0],)
        else:
            arg_str = ''

        # TODO: Should increment

        li = linestarts.get(opoffset, None)
        if li is not None and self.cur_line != li:
          self.cur_line = li

        debug('%s%d: %s%s (line %s)', indent, opoffset, byteName, arg_str,
            self.cur_line)
        debug('  %sval stack: %s', indent, stack_rep)
        debug('  %sblock stack: %s', indent, block_stack_rep)
              self.cur_line)
        if self.repr_ok:
            debug('  %sval stack: %s', indent, stack_rep)
        #debug('  %sblock stack: %s', indent, block_stack_rep)
        debug('')

    def dispatch(self, byteName, arguments):
@@ -206,9 +216,15 @@ def dispatch(self, byteName, arguments):
                why = bytecode_fn(*arguments)

        except:
            # deal with exceptions encountered while executing the op.
            # Deal with exceptions encountered while executing the op.
            self.last_exception = sys.exc_info()[:2] + (None,)
            log.info("Caught exception during execution")

            # NOTE: Why doesn't byterun use this info?
            #tb = sys.exc_info()[2]
            #traceback.print_tb(tb)

            debug1("Caught exception during execution of %s: %d", byteName,
                   len(self.frames))
            why = 'exception'
            self.except_frames = list(self.frames)

@@ -230,6 +246,12 @@ def run_frame(self, frame):
        """Run a frame until it returns or raises an exception.

        This function raises GuestException or returns the return value.

        Corresponds to PyEval_EvalFrameEx in ceval.c.  That returns 'PyObject*
        retval' -- but how does it indicate an exception?

        I think retval is NULL, and then

        """
        # bytecode offset -> line number
        #print('frame %s ' % frame)
@@ -249,6 +271,10 @@ def run_frame(self, frame):

            # When unwinding the block stack, we need to keep track of why we
            # are doing it.

            # NOTE: In addition to returning why == 'exception', this can also
            # RAISE GuestException from recursive call via call_function.

            why = self.dispatch(byteName, arguments)
            if why == 'exception':
                # TODO: ceval calls PyTraceBack_Here, not sure what that does.
@@ -258,6 +284,12 @@ def run_frame(self, frame):
                why = 'exception'

            if why != 'yield':

                # NOTE: why is used in a frame INTERNALLY after bytecode dispatch.
                # But what about ACROSS frames.  We need to unwind the call
                # stack too!  How is that done?
                # I don't want it to be done with GuestException!

                while why and frame.block_stack:
                    debug('WHY %s', why)
                    debug('STACK %s', frame.block_stack)
@@ -272,16 +304,22 @@ def run_frame(self, frame):

        if why == 'exception':
            exctype, value, tb = self.last_exception
            debug('exctype: %s' % exctype)
            debug('value: %s' % value)
            debug('unused tb: %s' % tb)

            #debug('exctype: %s' % exctype)
            #debug('value: %s' % value)
            #debug('unused tb: %s' % tb)

            if self.more_info:
                # Raise an exception with the EMULATED (guest) stack frames.
                raise GuestException(exctype, value, self.except_frames)
                # Recursive function calls can cause this I guess.
                if isinstance(value, GuestException):
                    raise value
                else:
                    # Raise an exception with the EMULATED (guest) stack frames.
                    raise GuestException(exctype, value, self.except_frames)
            else:
                raise exctype, value, tb

        #print('num_ticks: %d' % num_ticks)
        #debug1('num_ticks: %d' % num_ticks)
        return self.return_value

    def check_invariants(self):
@@ -490,6 +528,8 @@ def byte_COMPARE_OP(self, opnum):

    def byte_LOAD_ATTR(self, attr):
        obj = self.pop()
        #debug1('obj=%s, attr=%s', obj, attr)
        #debug1('dir(obj)=%s', dir(obj))
        val = getattr(obj, attr)
        self.push(val)

@@ -817,8 +857,10 @@ def call_function(self, arg, args, kwargs):
        posargs = self.popn(lenPos)
        posargs.extend(args)

        #debug('*** call_function stack = %s', self.frame.stack)

        func = self.pop()
        debug('*** call_function POPPED %s', func)
        #debug1('*** call_function POPPED %s', func)
        if getattr(func, 'func_name', None) == 'decode_next':
            raise AssertionError('BAD: %s' % func)

@@ -845,11 +887,12 @@ def call_function(self, arg, args, kwargs):
                    # More informative error that shows the frame.
                    raise TypeError(
                        'unbound method %s() must be called with %s instance '
                        'as first argument, was called with %s instance'
                        'as first argument, was called with %s instance '
                        '(frame: %s)' % (
                            func.im_func.func_name,
                            func.im_class.__name__,
                            type(posargs[0]).__name__,
                            #posargs[0],
                            self.frame,
                        )
                    )
@@ -881,17 +924,20 @@ def call_function(self, arg, args, kwargs):
        # next() and send()  that is a native python function.  We dO NOt need
        # to wrap it.

        interpret_bytecode = False
        do_wrap = False
        #debug1('FUNC %s', dir(func))
        if isinstance(func, types.FunctionType):
          interpret_bytecode = True
          do_wrap = True

        # Hack for case #4.
        if getattr(func, '__doc__', None) == 'DO_NOT_INTERPRET':
          interpret_bytecode = False
          do_wrap = False
          #raise AssertionError

        if interpret_bytecode:
            #debug1('*** WRAPPING %s', func)
        #debug1('do_wrap: %s', do_wrap)

        if do_wrap:
            debug1('*** WRAPPING %s', func)
            #debug1('%s', dir(func))
            #debug1('__doc__ %s', func.__doc__)

@@ -903,6 +949,7 @@ def call_function(self, arg, args, kwargs):
        else:
            byterun_func = func
         
        #debug1('  Calling: %s', byterun_func)
        retval = byterun_func(*posargs, **namedargs)
        self.push(retval)

@@ -943,6 +990,9 @@ def byte_YIELD_FROM(self):
    def byte_IMPORT_NAME(self, name):
        level, fromlist = self.popn(2)
        frame = self.frame

        # NOTE: This can read .pyc files not compiled with OPy!

        mod = __import__(name, frame.f_globals, frame.f_locals, fromlist, level)
        #print('-- IMPORTED %s -> %s' % (name, mod))
        self.push(mod)