diff --git a/src/RestrictedPython/Limits.py b/src/RestrictedPython/Limits.py index 2b984ea..0da8aad 100644 --- a/src/RestrictedPython/Limits.py +++ b/src/RestrictedPython/Limits.py @@ -14,7 +14,7 @@ limited_builtins = {} -def _limited_range(iFirst, *args): +def limited_range(iFirst, *args): # limited range function from Martijn Pieters RANGELIMIT = 1000 if not len(args): @@ -35,22 +35,22 @@ def _limited_range(iFirst, *args): return range(iStart, iEnd, iStep) -limited_builtins['range'] = _limited_range +limited_builtins['range'] = limited_range -def _limited_list(seq): +def limited_list(seq): if isinstance(seq, str): raise TypeError('cannot convert string to list') return list(seq) -limited_builtins['list'] = _limited_list +limited_builtins['list'] = limited_list -def _limited_tuple(seq): +def limited_tuple(seq): if isinstance(seq, str): raise TypeError('cannot convert string to tuple') return tuple(seq) -limited_builtins['tuple'] = _limited_tuple +limited_builtins['tuple'] = limited_tuple diff --git a/src/RestrictedPython/RCompile.py b/src/RestrictedPython/RCompile.py index c17d7d3..017e96e 100644 --- a/src/RestrictedPython/RCompile.py +++ b/src/RestrictedPython/RCompile.py @@ -14,11 +14,12 @@ Python standard library. """ -from compiler import ast as c_ast -from compiler import misc as c_misc -from compiler import parse as c_parse -from compiler import syntax as c_syntax +from compile import CompileResult +from compiler import ast +from compiler import misc +from compiler import parse from compiler import pycodegen +from compiler import syntax from compiler.pycodegen import AbstractCompileMode from compiler.pycodegen import Expression from compiler.pycodegen import findOp @@ -26,20 +27,18 @@ from compiler.pycodegen import Interactive from compiler.pycodegen import Module from compiler.pycodegen import ModuleCodeGenerator -from RestrictedPython import CompileResult -from RestrictedPython import MutatingWalker -from RestrictedPython.RestrictionMutator import RestrictionMutator +from RestrictionMutator import RestrictionMutator +import MutatingWalker -def _niceParse(source, filename, mode): + +def niceParse(source, filename, mode): if isinstance(source, unicode): # Use the utf-8-sig BOM so the compiler # detects this as a UTF-8 encoded string. source = '\xef\xbb\xbf' + source.encode('utf-8') try: - compiler_code = c_parse(source, mode) - # ast_code = ast.parse(source, filename, mode) - return compiler_code + return parse(source, mode) except: # Try to make a clean error message using # the builtin Python compiler. @@ -62,17 +61,16 @@ def __init__(self, source, filename): AbstractCompileMode.__init__(self, source, filename) def parse(self): - code = _niceParse(self.source, self.filename, self.mode) - return code + return niceParse(self.source, self.filename, self.mode) def _get_tree(self): - c_tree = self.parse() - MutatingWalker.walk(c_tree, self.rm) + tree = self.parse() + MutatingWalker.walk(tree, self.rm) if self.rm.errors: raise SyntaxError(self.rm.errors[0]) - c_misc.set_filename(self.filename, c_tree) - c_syntax.check(c_tree) - return c_tree + misc.set_filename(self.filename, tree) + syntax.check(tree) + return tree def compile(self): tree = self._get_tree() @@ -80,7 +78,7 @@ def compile(self): self.code = gen.getCode() -def _compileAndTuplize(gen): +def compileAndTuplize(gen): try: gen.compile() except TypeError as v: @@ -104,22 +102,22 @@ def compile_restricted_function(p, body, name, filename, globalize=None): appeared in a global statement at the top of the function). """ gen = RFunction(p, body, name, filename, globalize) - return _compileAndTuplize(gen) + return compileAndTuplize(gen) def compile_restricted_exec(source, filename=''): """Compiles a restricted code suite.""" gen = RModule(source, filename) - return _compileAndTuplize(gen) + return compileAndTuplize(gen) def compile_restricted_eval(source, filename=''): """Compiles a restricted expression.""" gen = RExpression(source, filename) - return _compileAndTuplize(gen) + return compileAndTuplize(gen) -def compile_restricted(source, filename, mode): # OLD +def compile_restricted(source, filename, mode): """Replacement for the builtin compile() function.""" if mode == "single": gen = RInteractive(source, filename) @@ -249,17 +247,17 @@ def __init__(self, p, body, name, filename, globals): def parse(self): # Parse the parameters and body, then combine them. firstline = 'def f(%s): pass' % self.params - tree = _niceParse(firstline, '', 'exec') + tree = niceParse(firstline, '', 'exec') f = tree.node.nodes[0] - body_code = _niceParse(self.body, self.filename, 'exec') + body_code = niceParse(self.body, self.filename, 'exec') # Stitch the body code into the function. f.code.nodes = body_code.node.nodes f.name = self.name # Look for a docstring, if there are any nodes at all if len(f.code.nodes) > 0: stmt1 = f.code.nodes[0] - if (isinstance(stmt1, c_ast.Discard) and - isinstance(stmt1.expr, c_ast.Const) and + if (isinstance(stmt1, ast.Discard) and + isinstance(stmt1.expr, ast.Const) and isinstance(stmt1.expr.value, str)): f.doc = stmt1.expr.value # The caller may specify that certain variables are globals @@ -267,5 +265,5 @@ def parse(self): # The only known example is the variables context, container, # script, traverse_subpath in PythonScripts. if self.globals: - f.code.nodes.insert(0, c_ast.Global(self.globals)) + f.code.nodes.insert(0, ast.Global(self.globals)) return tree diff --git a/src/RestrictedPython/SelectCompiler.py b/src/RestrictedPython/SelectCompiler.py index a459f3d..ad0d80d 100644 --- a/src/RestrictedPython/SelectCompiler.py +++ b/src/RestrictedPython/SelectCompiler.py @@ -18,10 +18,10 @@ from compiler.consts import OP_ASSIGN from compiler.consts import OP_DELETE from compiler.transformer import parse -from RestrictedPython.RCompile import compile_restricted -from RestrictedPython.RCompile import compile_restricted_eval -from RestrictedPython.RCompile import compile_restricted_exec -from RestrictedPython.RCompile import compile_restricted_function +from RCompile import compile_restricted +from RCompile import compile_restricted_eval +from RCompile import compile_restricted_exec +from RCompile import compile_restricted_function # Use the compiler from the standard library. import compiler diff --git a/src/RestrictedPython/Utilities.py b/src/RestrictedPython/Utilities.py index aee2b20..160fecd 100644 --- a/src/RestrictedPython/Utilities.py +++ b/src/RestrictedPython/Utilities.py @@ -25,6 +25,12 @@ utility_builtins['set'] = set utility_builtins['frozenset'] = frozenset +try: + import sets + utility_builtins['sets'] = sets +except ImportError: + pass + try: import DateTime utility_builtins['DateTime'] = DateTime.DateTime diff --git a/src/RestrictedPython/test_helper.py b/src/RestrictedPython/test_helper.py deleted file mode 100644 index 199c9c5..0000000 --- a/src/RestrictedPython/test_helper.py +++ /dev/null @@ -1,172 +0,0 @@ -############################################################################## -# -# Copyright (c) 2003 Zope Foundation and Contributors. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE -# -############################################################################## -"""Verify simple properties of bytecode. - -Some of the transformations performed by the RestrictionMutator are -tricky. This module checks the generated bytecode as a way to verify -the correctness of the transformations. Violations of some -restrictions are obvious from inspection of the bytecode. For -example, the bytecode should never contain a LOAD_ATTR call, because -all attribute access is performed via the _getattr_() checker -function. -""" - -from dis import findlinestarts - -import dis -import types - - -def verify(code): - """Verify all code objects reachable from code. - - In particular, traverse into contained code objects in the - co_consts table. - """ - _verifycode(code) - for ob in code.co_consts: - if isinstance(ob, types.CodeType): - verify(ob) - - -def _verifycode(code): - line = code.co_firstlineno - # keep a window of the last three opcodes, with the most recent first - window = (None, None, None) - with_context = (None, None) - - for op in _disassemble(code): - if op.line is not None: - line = op.line - if op.opname.endswith("LOAD_ATTR"): - # All the user code that generates LOAD_ATTR should be - # rewritten, but the code generated for a list comp - # includes a LOAD_ATTR to extract the append method. - # Another exception is the new-in-Python 2.6 'context - # managers', which do a LOAD_ATTR for __exit__ and - # __enter__. - if op.arg == "__exit__": - with_context = (op, with_context[1]) - elif op.arg == "__enter__": - with_context = (with_context[0], op) - elif not ((op.arg == "__enter__" and - window[0].opname == "ROT_TWO" and - window[1].opname == "DUP_TOP") or - (op.arg == "append" and - window[0].opname == "DUP_TOP" and - window[1].opname == "BUILD_LIST")): - raise ValueError("direct attribute access %s: %s, %s:%d" - % (op.opname, op.arg, code.co_filename, line)) - if op.opname in ("WITH_CLEANUP"): - # Here we check if the LOAD_ATTR for __exit__ and - # __enter__ were part of a 'with' statement by checking - # for the 'WITH_CLEANUP' bytecode. If one is seen, we - # clear the with_context variable and let it go. The - # access was safe. - with_context = (None, None) - if op.opname in ("STORE_ATTR", "DEL_ATTR"): - if not (window[0].opname == "CALL_FUNCTION" and - window[2].opname == "LOAD_GLOBAL" and - window[2].arg == "_write_"): - # check that arg is appropriately wrapped - for i, op in enumerate(window): - print(i, op.opname, op.arg) - raise ValueError("unguard attribute set/del at %s:%d" - % (code.co_filename, line)) - if op.opname.startswith("UNPACK"): - # An UNPACK opcode extracts items from iterables, and that's - # unsafe. The restricted compiler doesn't remove UNPACK opcodes, - # but rather *inserts* a call to _getiter_() before each, and - # that's the pattern we need to see. - if not (window[0].opname == "CALL_FUNCTION" and - window[1].opname == "ROT_TWO" and - window[2].opname == "LOAD_GLOBAL" and - window[2].arg == "_getiter_"): - raise ValueError("unguarded unpack sequence at %s:%d" % - (code.co_filename, line)) - - # should check CALL_FUNCTION_{VAR,KW,VAR_KW} but that would - # require a potentially unlimited history. need to refactor - # the "window" before I can do that. - - if op.opname == "LOAD_SUBSCR": - raise ValueError("unguarded index of sequence at %s:%d" % - (code.co_filename, line)) - - window = (op,) + window[:2] - - if not with_context == (None, None): - # An access to __enter__ and __exit__ was performed but not as - # part of a 'with' statement. This is not allowed. - for op in with_context: - if op is not None: - if op.line is not None: - line = op.line - raise ValueError("direct attribute access %s: %s, %s:%d" - % (op.opname, op.arg, code.co_filename, line)) - - -class Op(object): - __slots__ = ( - "opname", # string, name of the opcode - "argcode", # int, the number of the argument - "arg", # any, the object, name, or value of argcode - "line", # int, line number or None - "target", # boolean, is this op the target of a jump - "pos", # int, offset in the bytecode - ) - - def __init__(self, opcode, pos): - self.opname = dis.opname[opcode] - self.arg = None - self.line = None - self.target = False - self.pos = pos - - -def _disassemble(co, lasti=-1): - code = co.co_code - labels = dis.findlabels(code) - linestarts = dict(findlinestarts(co)) - n = len(code) - i = 0 - extended_arg = 0 - free = co.co_cellvars + co.co_freevars - while i < n: - op = ord(code[i]) - o = Op(op, i) - i += 1 - if i in linestarts and i > 0: - o.line = linestarts[i] - if i in labels: - o.target = True - if op > dis.HAVE_ARGUMENT: - arg = ord(code[i]) + ord(code[i + 1]) * 256 + extended_arg - extended_arg = 0 - i += 2 - if op == dis.EXTENDED_ARG: - extended_arg = arg << 16 - o.argcode = arg - if op in dis.hasconst: - o.arg = co.co_consts[arg] - elif op in dis.hasname: - o.arg = co.co_names[arg] - elif op in dis.hasjrel: - o.arg = i + arg - elif op in dis.haslocal: - o.arg = co.co_varnames[arg] - elif op in dis.hascompare: - o.arg = dis.cmp_op[arg] - elif op in dis.hasfree: - o.arg = free[arg] - yield o diff --git a/src/RestrictedPython/tests/testCompile.py b/src/RestrictedPython/tests/testCompile.py index 3b145e2..859b20d 100644 --- a/src/RestrictedPython/tests/testCompile.py +++ b/src/RestrictedPython/tests/testCompile.py @@ -12,8 +12,7 @@ # ############################################################################## -# Need to be ported -from RestrictedPython.RCompile import _niceParse +from RestrictedPython.RCompile import niceParse import compiler.ast import unittest @@ -25,11 +24,11 @@ def testUnicodeSource(self): # We support unicode sourcecode. source = u"u'Ä väry nice säntänce with umlauts.'" - parsed = _niceParse(source, "test.py", "exec") + parsed = niceParse(source, "test.py", "exec") self.failUnless(isinstance(parsed, compiler.ast.Module)) - parsed = _niceParse(source, "test.py", "single") + parsed = niceParse(source, "test.py", "single") self.failUnless(isinstance(parsed, compiler.ast.Module)) - parsed = _niceParse(source, "test.py", "eval") + parsed = niceParse(source, "test.py", "eval") self.failUnless(isinstance(parsed, compiler.ast.Expression)) diff --git a/src/RestrictedPython/tests/testRestrictions.py b/src/RestrictedPython/tests/testRestrictions.py index 7f3a383..7527ffc 100644 --- a/src/RestrictedPython/tests/testRestrictions.py +++ b/src/RestrictedPython/tests/testRestrictions.py @@ -9,8 +9,8 @@ from RestrictedPython.RCompile import compile_restricted from RestrictedPython.RCompile import RFunction from RestrictedPython.RCompile import RModule -from RestrictedPython.test_helper import verify from RestrictedPython.tests import restricted_module +from RestrictedPython.tests import verify import os import re @@ -92,7 +92,7 @@ def create_rmodule(): 'len', 'chr', 'ord', ): rmodule[name] = builtins[name] - exec(code, rmodule) + exec code in rmodule class AccessDenied (Exception): @@ -218,12 +218,18 @@ def inplacevar_wrapper(op, x, y): class RestrictionTests(unittest.TestCase): def execFunc(self, name, *args, **kw): func = rmodule[name] - verify(func.func_code) + verify.verify(func.func_code) func.func_globals.update({'_getattr_': guarded_getattr, '_getitem_': guarded_getitem, '_write_': TestGuard, '_print_': PrintCollector, - '_getiter_': iter, + # I don't want to write something as involved as ZopeGuard's + # SafeIter just for these tests. Using the builtin list() function + # worked OK for everything the tests did at the time this was added, + # but may fail in the future. If Python 2.1 is no longer an + # interesting platform then, using 2.2's builtin iter() here should + # work for everything. + '_getiter_': list, '_apply_': apply_wrapper, '_inplacevar_': inplacevar_wrapper, }) @@ -323,7 +329,7 @@ def _compile_file(self, name): f.close() co = compile_restricted(source, path, "exec") - verify(co) + verify.verify(co) return co def test_UnpackSequence(self): @@ -367,7 +373,7 @@ def getiter(seq): def test_UnpackSequenceExpression(self): co = compile_restricted("[x for x, y in [(1, 2)]]", "", "eval") - verify(co) + verify.verify(co) calls = [] def getiter(s): @@ -379,7 +385,7 @@ def getiter(s): def test_UnpackSequenceSingle(self): co = compile_restricted("x, y = 1, 2", "", "single") - verify(co) + verify.verify(co) calls = [] def getiter(s): diff --git a/tests/builtins/test_limits.py b/tests/builtins/test_limits.py index 9f7fd41..ac034be 100644 --- a/tests/builtins/test_limits.py +++ b/tests/builtins/test_limits.py @@ -1,72 +1,72 @@ -from RestrictedPython.Limits import _limited_list -from RestrictedPython.Limits import _limited_range -from RestrictedPython.Limits import _limited_tuple +from RestrictedPython.Limits import limited_list +from RestrictedPython.Limits import limited_range +from RestrictedPython.Limits import limited_tuple import pytest -def test__limited_range_length_1(): - result = _limited_range(1) +def test_limited_range_length_1(): + result = limited_range(1) assert result == range(0, 1) -def test__limited_range_length_10(): - result = _limited_range(10) +def test_limited_range_length_10(): + result = limited_range(10) assert result == range(0, 10) -def test__limited_range_5_10(): - result = _limited_range(5, 10) +def test_limited_range_5_10(): + result = limited_range(5, 10) assert result == range(5, 10) -def test__limited_range_5_10_sm1(): - result = _limited_range(5, 10, -1) +def test_limited_range_5_10_sm1(): + result = limited_range(5, 10, -1) assert result == range(5, 10, -1) -def test__limited_range_15_10_s2(): - result = _limited_range(15, 10, 2) +def test_limited_range_15_10_s2(): + result = limited_range(15, 10, 2) assert result == range(15, 10, 2) -def test__limited_range_no_input(): +def test_limited_range_no_input(): with pytest.raises(TypeError): - _limited_range() + limited_range() -def test__limited_range_more_steps(): +def test_limited_range_more_steps(): with pytest.raises(AttributeError): - _limited_range(0, 0, 0, 0) + limited_range(0, 0, 0, 0) -def test__limited_range_zero_step(): +def test_limited_range_zero_step(): with pytest.raises(ValueError): - _limited_range(0, 10, 0) + limited_range(0, 10, 0) -def test__limited_range_range_overflow(): +def test_limited_range_range_overflow(): with pytest.raises(ValueError): - _limited_range(0, 5000, 1) + limited_range(0, 5000, 1) -def test__limited_list_valid_list_input(): +def test_limited_list_valid_list_input(): input = [1, 2, 3] - result = _limited_list(input) + result = limited_list(input) assert result == input -def test__limited_list_invalid_string_input(): +def test_limited_list_invalid_string_input(): with pytest.raises(TypeError): - _limited_list('input') + limited_list('input') -def test__limited_tuple_valid_list_input(): +def test_limited_tuple_valid_list_input(): input = [1, 2, 3] - result = _limited_tuple(input) + result = limited_tuple(input) assert result == tuple(input) -def test__limited_tuple_invalid_string_input(): +def test_limited_tuple_invalid_string_input(): with pytest.raises(TypeError): - _limited_tuple('input') + limited_tuple('input') diff --git a/tests/builtins/test_utilities.py b/tests/builtins/test_utilities.py index d6b0426..d6f5dba 100644 --- a/tests/builtins/test_utilities.py +++ b/tests/builtins/test_utilities.py @@ -1,3 +1,6 @@ +from RestrictedPython._compat import IS_PY3 +import pytest + def test_string_in_utility_builtins(): import string @@ -28,6 +31,14 @@ def test_set_in_utility_builtins(): assert utility_builtins['set'] is set +@pytest.mark.skipif(IS_PY3, + reason='Python 3 has no longer includes the sets module.') +def test_sets_in_utility_builtins(): + from RestrictedPython.Utilities import utility_builtins + import sets + assert utility_builtins['sets'] is sets + + def test_frozenset_in_utility_builtins(): from RestrictedPython.Utilities import utility_builtins assert utility_builtins['frozenset'] is frozenset