From d5c87872a8be37aeb3454f8e4db1c60da76ca0f8 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Fri, 3 Feb 2017 18:02:30 +0100 Subject: [PATCH 01/16] more docs --- README.rst | 7 ++ README.txt | 1 - CHANGES.rst => docs/CHANGES.rst | 0 docs/contributing/index.rst | 4 + docs/index.rst | 5 + docs/install/index.rst | 10 ++ docs/roadmap/index.rst | 4 + docs/usage/api.rst | 22 +++++ docs/usage/basic_usage.rst | 81 +++++++++++++++ docs/usage/framework_usage.rst | 61 ++++++++++++ docs/usage/index.rst | 169 +------------------------------- setup.py | 6 +- 12 files changed, 201 insertions(+), 169 deletions(-) create mode 100644 README.rst delete mode 100644 README.txt rename CHANGES.rst => docs/CHANGES.rst (100%) create mode 100644 docs/contributing/index.rst create mode 100644 docs/install/index.rst create mode 100644 docs/roadmap/index.rst create mode 100644 docs/usage/api.rst create mode 100644 docs/usage/basic_usage.rst create mode 100644 docs/usage/framework_usage.rst diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..7894bcc --- /dev/null +++ b/README.rst @@ -0,0 +1,7 @@ +================ +RestrictedPython +================ + +RestrictedPython is a defined subset of the Python language which allows to provide a program input into a trusted environment. + +For full documentation please see docs/index. diff --git a/README.txt b/README.txt deleted file mode 100644 index 03610af..0000000 --- a/README.txt +++ /dev/null @@ -1 +0,0 @@ -Please refer to src/RestrictedPython/README.txt. diff --git a/CHANGES.rst b/docs/CHANGES.rst similarity index 100% rename from CHANGES.rst rename to docs/CHANGES.rst diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst new file mode 100644 index 0000000..da33d08 --- /dev/null +++ b/docs/contributing/index.rst @@ -0,0 +1,4 @@ +Contributing +============ + +https://trello.com/b/pKaXJIlT/restrictedpython diff --git a/docs/index.rst b/docs/index.rst index 253d047..2459200 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,6 +25,11 @@ Contents upgrade/index upgrade_dependencies/index + roadmap/index + contributing/index + + CHANGES + Indices and tables ================== diff --git a/docs/install/index.rst b/docs/install/index.rst new file mode 100644 index 0000000..0286941 --- /dev/null +++ b/docs/install/index.rst @@ -0,0 +1,10 @@ +Install / Depend on RestrictedPython +==================================== + +RestrictedPython is usually not used stand alone, if you use it in context of your package add it to ``install_requires`` in your ``setup.py`` or a ``requirement.txt`` used by ``pip``. + +For a standalone usage: + +.. code:: bash + + pip install RestrictedPython diff --git a/docs/roadmap/index.rst b/docs/roadmap/index.rst new file mode 100644 index 0000000..379b54c --- /dev/null +++ b/docs/roadmap/index.rst @@ -0,0 +1,4 @@ +Roadmap for RestrictedPython +============================ + +https://trello.com/b/pKaXJIlT/restrictedpython diff --git a/docs/usage/api.rst b/docs/usage/api.rst new file mode 100644 index 0000000..90195bc --- /dev/null +++ b/docs/usage/api.rst @@ -0,0 +1,22 @@ +API overview +------------ + +RestrictedPython has tree major scopes: + +1. ``compile_restricted`` methods: + + * ``compile_restricted`` + * ``compile_restricted_exec`` + * ``compile_restricted_eval`` + * ``compile_restricted_single`` + * ``compile_restricted_function`` + +2. restricted builtins + + * ``safe_builtins`` + * ``limited_builtins`` + * ``utility_builtins`` + +3. helper modules + + * ``PrintCollector`` diff --git a/docs/usage/basic_usage.rst b/docs/usage/basic_usage.rst new file mode 100644 index 0000000..522c23f --- /dev/null +++ b/docs/usage/basic_usage.rst @@ -0,0 +1,81 @@ +Basic usage +----------- + +The general workflow to execute Python code that is loaded within a Python program is: + +.. code:: Python + + source_code = """ + def do_something(): + pass + """ + + byte_code = compile(source_code, filename='', mode='exec') + exec(byte_code) + do_something() + +With RestrictedPython that workflow should be as straight forward as possible: + +.. code:: Python + + from RestrictedPython import compile_restricted as compile + + source_code = """ + def do_something(): + pass + """ + + byte_code = compile(source_code, filename='', mode='exec') + exec(byte_code) + do_something() + +With that simple addition: + +.. code:: Python + + from RestrictedPython import compile_restricted as compile + +it uses a predefined policy that checks and modify the source code and checks against a restricted subset of the Python language. +The compiled source code is still executed against the full available set of library modules and methods. + +The Python :py:func:`exec` takes three parameters: + +* ``code`` which is the compiled byte code +* ``globals`` which is global dictionary +* ``locals`` which is the local dictionary + +By limiting the entries in the ``globals`` and ``locals`` dictionaries you +restrict the access to the available library modules and methods. + +Providing defined dictionaries for ``exec()`` should be used in context of RestrictedPython. + +.. code:: Python + + byte_code = + exec(byte_code, { ... }, { ... }) + +Typically there is a defined set of allowed modules, methods and constants used in that context. +RestrictedPython provides three predefined built-ins for that: + +* ``safe_builtins`` +* ``limited_builtins`` +* ``utilities_builtins`` + +So you normally end up using: + +.. code:: Python + + from RestrictedPython import ..._builtins + from RestrictedPython import compile_restricted as compile + + source_code = """""" + + try: + byte_code = compile(source_code, filename='', mode='exec') + + used_builtins = ..._builtins + { } + exec(byte_code, used_buildins, None) + except SyntaxError as e: + ... + +One common advanced usage would be to define an own restricted builtin dictionary. diff --git a/docs/usage/framework_usage.rst b/docs/usage/framework_usage.rst new file mode 100644 index 0000000..5d5e1ed --- /dev/null +++ b/docs/usage/framework_usage.rst @@ -0,0 +1,61 @@ +.. _sec_usage_frameworks: + +Usage in frameworks and Zope +---------------------------- + +One major issue with using ``compile_restricted`` directly in a framework is, that you have to use try-except statements to handle problems and it might be a bit harder to provide useful information to the user. +RestrictedPython provides four specialized compile_restricted methods: + +* ``compile_restricted_exec`` +* ``compile_restricted_eval`` +* ``compile_restricted_single`` +* ``compile_restricted_function`` + +Those four methods return a tuple with four elements: + +* ``byte_code`` object or ``None`` if ``errors`` is not empty +* ``errors`` a tuple with error messages +* ``warnings`` a list with warnings +* ``used_names`` a set / dictionary with collected used names of library calls + +Those three information "lists" could be used to provide the user with informations about the compiled source code. + +Typical uses cases for the four specialized methods: + +* ``compile_restricted_exec`` --> Python Modules or Scripts that should be used or called by the framework itself or from user calls +* ``compile_restricted_eval`` --> Templates +* ``compile_restricted_single`` +* ``compile_restricted_function`` + +Modifying the builtins is straight forward, it is just a dictionary containing access pointers to available library elements. +Modification is normally removing elements from existing builtins or adding allowed elements by copying from globals. + +For frameworks it could possibly also be useful to change handling of specific Python language elements. +For that use case RestrictedPython provide the possibility to pass an own policy. +A policy is basically a special ``NodeTransformer`` that could be instantiated with three params for ``errors``, ``warnings`` and ``used_names``, it should be a subclass of RestrictingNodeTransformer (that subclassing will maybe later be enforced). + +.. code:: Python + + OwnRestrictingNodeTransformer(errors=[], warnings=[], used_names=[]) + +One special case (defined to unblock ports of Zope Packages to Python 3) is to actually use RestrictedPython in an unrestricted mode, by providing a Null-Policy (aka ``None``). + +All ``compile_restricted*`` methods do have a optional parameter ``policy``, where a specific policy could be provided. + +.. code:: Python + + source_code = """""" + + policy = OwnRestrictingNodeTransformer + + byte_code = compile(source_code, filename='', mode='exec', policy=policy) + exec(byte_code, { ... }, { ... }) + +The Special case "unrestricted RestrictedPython" would be: + +.. code:: Python + + source_code = """""" + + byte_code = compile(source_code, filename='', mode='exec', policy=None) + exec(byte_code, globals(), None) diff --git a/docs/usage/index.rst b/docs/usage/index.rst index 5079e0d..48bd250 100644 --- a/docs/usage/index.rst +++ b/docs/usage/index.rst @@ -1,169 +1,6 @@ Usage of RestrictedPython ========================= -API overview ------------- - -RestrictedPython has tree major scopes: - -1. ``compile_restricted`` methods: - - * ``compile_restricted`` - * ``compile_restricted_exec`` - * ``compile_restricted_eval`` - * ``compile_restricted_single`` - * ``compile_restricted_function`` - -2. restricted builtins - - * ``safe_builtins`` - * ``limited_builtins`` - * ``utility_builtins`` - -3. helper modules - - * ``PrintCollector`` - -Basic usage ------------ - -The general workflow to execute Python code that is loaded within a Python program is: - -.. code:: Python - - source_code = """ - def do_something(): - pass - """ - - byte_code = compile(source_code, filename='', mode='exec') - exec(byte_code) - do_something() - -With RestrictedPython that workflow should be as straight forward as possible: - -.. code:: Python - - from RestrictedPython import compile_restricted as compile - - source_code = """ - def do_something(): - pass - """ - - byte_code = compile(source_code, filename='', mode='exec') - exec(byte_code) - do_something() - -With that simple addition: - -.. code:: Python - - from RestrictedPython import compile_restricted as compile - -it uses a predefined policy that checks and modify the source code and checks against a restricted subset of the Python language. -The compiled source code is still executed against the full available set of library modules and methods. - -The Python :py:func:`exec` takes three parameters: - -* ``code`` which is the compiled byte code -* ``globals`` which is global dictionary -* ``locals`` which is the local dictionary - -By limiting the entries in the ``globals`` and ``locals`` dictionaries you -restrict the access to the available library modules and methods. - -Providing defined dictionaries for ``exec()`` should be used in context of RestrictedPython. - -.. code:: Python - - byte_code = - exec(byte_code, { ... }, { ... }) - -Typically there is a defined set of allowed modules, methods and constants used in that context. -RestrictedPython provides three predefined built-ins for that: - -* ``safe_builtins`` -* ``limited_builtins`` -* ``utilities_builtins`` - -So you normally end up using: - -.. code:: Python - - from RestrictedPython import ..._builtins - from RestrictedPython import compile_restricted as compile - - source_code = """""" - - try: - byte_code = compile(source_code, filename='', mode='exec') - - used_builtins = ..._builtins + { } - exec(byte_code, used_buildins, None) - except SyntaxError as e: - ... - -One common advanced usage would be to define an own restricted builtin dictionary. - -.. _sec_usage_frameworks: - -Usage in frameworks and Zope ----------------------------- - -One major issue with using ``compile_restricted`` directly in a framework is, that you have to use try-except statements to handle problems and it might be a bit harder to provide useful information to the user. -RestrictedPython provides four specialized compile_restricted methods: - -* ``compile_restricted_exec`` -* ``compile_restricted_eval`` -* ``compile_restricted_single`` -* ``compile_restricted_function`` - -Those four methods return a tuple with four elements: - -* ``byte_code`` object or ``None`` if ``errors`` is not empty -* ``errors`` a tuple with error messages -* ``warnings`` a list with warnings -* ``used_names`` a set / dictionary with collected used names of library calls - -Those three information "lists" could be used to provide the user with informations about the compiled source code. - -Typical uses cases for the four specialized methods: - -* ``compile_restricted_exec`` --> Python Modules or Scripts that should be used or called by the framework itself or from user calls -* ``compile_restricted_eval`` --> Templates -* ``compile_restricted_single`` -* ``compile_restricted_function`` - -Modifying the builtins is straight forward, it is just a dictionary containing access pointers to available library elements. -Modification is normally removing elements from existing builtins or adding allowed elements by copying from globals. - -For frameworks it could possibly also be useful to change handling of specific Python language elements. -For that use case RestrictedPython provide the possibility to pass an own policy. -A policy is basically a special ``NodeTransformer`` that could be instantiated with three params for ``errors``, ``warnings`` and ``used_names``, it should be a subclass of RestrictingNodeTransformer (that subclassing will maybe later be enforced). - -.. code:: Python - - OwnRestrictingNodeTransformer(errors=[], warnings=[], used_names=[]) - -One special case (defined to unblock ports of Zope Packages to Python 3) is to actually use RestrictedPython in an unrestricted mode, by providing a Null-Policy (aka ``None``). - -All ``compile_restricted*`` methods do have a optional parameter ``policy``, where a specific policy could be provided. - -.. code:: Python - - source_code = """""" - - policy = OwnRestrictingNodeTransformer - - byte_code = compile(source_code, filename='', mode='exec', policy=policy) - exec(byte_code, { ... }, { ... }) - -The Special case "unrestricted RestrictedPython" would be: - -.. code:: Python - - source_code = """""" - - byte_code = compile(source_code, filename='', mode='exec', policy=None) - exec(byte_code, globals(), None) +.. include:: api.rst +.. include:: basic_usage.rst +.. include:: framework_usage.rst diff --git a/setup.py b/setup.py index dfe1d49..99c9d0b 100644 --- a/setup.py +++ b/setup.py @@ -29,8 +29,10 @@ def read(*rnames): license='ZPL 2.1', description='RestrictedPython provides a restricted execution ' 'environment for Python, e.g. for running untrusted code.', - long_description=(read('src', 'RestrictedPython', 'README.rst') + '\n' + - read('CHANGES.rst')), + long_description=(read('README.rst') + '\n' + + read('docs', 'install', 'index.rst') + '\n' + + read('docs', 'usage', 'basic_usage.rst') + '\n' + + read('docs', 'CHANGES.rst')), classifiers=[ 'License :: OSI Approved :: Zope Public License', 'Programming Language :: Python', From 66d99539540d233b9f079edb4286f38b9f4c06c5 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 4 Feb 2017 15:19:01 +0100 Subject: [PATCH 02/16] well format ast for comparision --- docs/upgrade/ast/python2_6.ast | 61 ++++++-- docs/upgrade/ast/python2_7.ast | 56 +++++-- docs/upgrade/ast/python3_0.ast | 254 ++++++++++++++++-------------- docs/upgrade/ast/python3_1.ast | 256 ++++++++++++++++-------------- docs/upgrade/ast/python3_2.ast | 278 +++++++++++++++++---------------- docs/upgrade/ast/python3_3.ast | 105 ++++++++----- docs/upgrade/ast/python3_4.ast | 67 ++++---- docs/upgrade/ast/python3_5.ast | 125 +++++++++------ docs/upgrade/ast/python3_6.ast | 93 +++++------ 9 files changed, 733 insertions(+), 562 deletions(-) diff --git a/docs/upgrade/ast/python2_6.ast b/docs/upgrade/ast/python2_6.ast index d4af5b1..3ea12f8 100644 --- a/docs/upgrade/ast/python2_6.ast +++ b/docs/upgrade/ast/python2_6.ast @@ -81,20 +81,49 @@ module Python version "2.6" -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset) - expr_context = Load | Store | Del | AugLoad | AugStore | Param - - slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step) + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Ellipsis + | Slice(expr? lower, expr? upper, expr? step) | ExtSlice(slice* dims) | Index(expr value) - boolop = And | Or - - operator = Add | Sub | Mult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub - - cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn comprehension = (expr target, expr iter, expr* ifs) @@ -103,11 +132,11 @@ module Python version "2.6" attributes (int lineno, int col_offset) arguments = (expr* args, identifier? vararg, - identifier? kwarg, expr* defaults) + identifier? kwarg, expr* defaults) - -- keyword arguments supplied to call - keyword = (identifier arg, expr value) + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) - -- import name with optional 'as' alias. - alias = (identifier name, identifier? asname) + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) } diff --git a/docs/upgrade/ast/python2_7.ast b/docs/upgrade/ast/python2_7.ast index fc3aba1..899cdc5 100644 --- a/docs/upgrade/ast/python2_7.ast +++ b/docs/upgrade/ast/python2_7.ast @@ -11,7 +11,7 @@ module Python version "2.7" | Suite(stmt* body) stmt = FunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list) + stmt* body, expr* decorator_list) | ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list) | Return(expr? value) @@ -84,20 +84,48 @@ module Python version "2.7" -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset) - expr_context = Load | Store | Del | AugLoad | AugStore | Param + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param - slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step) + slice = Ellipsis + | Slice(expr? lower, expr? upper, expr? step) | ExtSlice(slice* dims) | Index(expr value) boolop = And | Or - operator = Add | Sub | Mult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub - - cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn comprehension = (expr target, expr iter, expr* ifs) @@ -106,11 +134,11 @@ module Python version "2.7" attributes (int lineno, int col_offset) arguments = (expr* args, identifier? vararg, - identifier? kwarg, expr* defaults) + identifier? kwarg, expr* defaults) - -- keyword arguments supplied to call - keyword = (identifier arg, expr value) + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) - -- import name with optional 'as' alias. - alias = (identifier name, identifier? asname) + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) } diff --git a/docs/upgrade/ast/python3_0.ast b/docs/upgrade/ast/python3_0.ast index 2241781..44d6e18 100644 --- a/docs/upgrade/ast/python3_0.ast +++ b/docs/upgrade/ast/python3_0.ast @@ -3,117 +3,145 @@ module Python version "3.0" { - mod = Module(stmt* body) - | Interactive(stmt* body) - | Expression(expr body) - - -- not really an actual node but useful in Jython's typesystem. - | Suite(stmt* body) - - stmt = FunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list, expr? returns) - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - expr? starargs, - expr? kwargs, - stmt* body, - expr *decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(expr context_expr, expr? optional_vars, stmt* body) - - | Raise(expr? exc, expr? cause) - | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) - | TryFinally(stmt* body, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? - expr = BoolOp(boolop op, expr* values) - | BinOp(expr left, operator op, expr right) - | UnaryOp(unaryop op, expr operand) - | Lambda(arguments args, expr body) - | IfExp(expr test, expr body, expr orelse) - | Dict(expr* keys, expr* values) - | Set(expr* elts) - | ListComp(expr elt, comprehension* generators) - | SetComp(expr elt, comprehension* generators) - | DictComp(expr key, expr value, comprehension* generators) - | GeneratorExp(expr elt, comprehension* generators) - -- the grammar constrains where yield expressions can occur - | Yield(expr? value) - -- need sequences for compare to distinguish between - -- x < 4 < 3 and (x < 4) < 3 - | Compare(expr left, cmpop* ops, expr* comparators) - | Call(expr func, expr* args, keyword* keywords, - expr? starargs, expr? kwargs) - | Num(object n) -- a number as a PyObject. - | Str(string s) -- need to specify raw, unicode, etc? - | Bytes(string s) - | Ellipsis - -- other literals? bools? - - -- the following expression can appear in assignment context - | Attribute(expr value, identifier attr, expr_context ctx) - | Subscript(expr value, slice slice, expr_context ctx) - | Starred(expr value, expr_context ctx) - | Name(identifier id, expr_context ctx) - | List(expr* elts, expr_context ctx) - | Tuple(expr* elts, expr_context ctx) - - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - expr_context = Load | Store | Del | AugLoad | AugStore | Param - - slice = Slice(expr? lower, expr? upper, expr? step) - | ExtSlice(slice* dims) - | Index(expr value) - - boolop = And | Or - - operator = Add | Sub | Mult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub - - cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn - - comprehension = (expr target, expr iter, expr* ifs) - - -- not sure what to call the first argument for raise and except - excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) - attributes (int lineno, int col_offset) - - arguments = (arg* args, identifier? vararg, expr? varargannotation, - arg* kwonlyargs, identifier? kwarg, - expr? kwargannotation, expr* defaults, - expr* kw_defaults) - arg = (identifier arg, expr? annotation) - - -- keyword arguments supplied to call - keyword = (identifier arg, expr value) - - -- import name with optional 'as' alias. - alias = (identifier name, identifier? asname) + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr *decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + | Raise(expr? exc, expr? cause) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(string s) + | Ellipsis + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, identifier? vararg, expr? varargannotation, + arg* kwonlyargs, identifier? kwarg, + expr? kwargannotation, expr* defaults, + expr* kw_defaults) + arg = (identifier arg, expr? annotation) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) } diff --git a/docs/upgrade/ast/python3_1.ast b/docs/upgrade/ast/python3_1.ast index 8b9dd80..a0bbfb6 100644 --- a/docs/upgrade/ast/python3_1.ast +++ b/docs/upgrade/ast/python3_1.ast @@ -3,117 +3,147 @@ module Python version "3.1" { - mod = Module(stmt* body) - | Interactive(stmt* body) - | Expression(expr body) - - -- not really an actual node but useful in Jython's typesystem. - | Suite(stmt* body) - - stmt = FunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list, expr? returns) - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - expr? starargs, - expr? kwargs, - stmt* body, - expr *decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(expr context_expr, expr? optional_vars, stmt* body) - - | Raise(expr? exc, expr? cause) - | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) - | TryFinally(stmt* body, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? - expr = BoolOp(boolop op, expr* values) - | BinOp(expr left, operator op, expr right) - | UnaryOp(unaryop op, expr operand) - | Lambda(arguments args, expr body) - | IfExp(expr test, expr body, expr orelse) - | Dict(expr* keys, expr* values) - | Set(expr* elts) - | ListComp(expr elt, comprehension* generators) - | SetComp(expr elt, comprehension* generators) - | DictComp(expr key, expr value, comprehension* generators) - | GeneratorExp(expr elt, comprehension* generators) - -- the grammar constrains where yield expressions can occur - | Yield(expr? value) - -- need sequences for compare to distinguish between - -- x < 4 < 3 and (x < 4) < 3 - | Compare(expr left, cmpop* ops, expr* comparators) - | Call(expr func, expr* args, keyword* keywords, - expr? starargs, expr? kwargs) - | Num(object n) -- a number as a PyObject. - | Str(string s) -- need to specify raw, unicode, etc? - | Bytes(string s) - | Ellipsis - -- other literals? bools? - - -- the following expression can appear in assignment context - | Attribute(expr value, identifier attr, expr_context ctx) - | Subscript(expr value, slice slice, expr_context ctx) - | Starred(expr value, expr_context ctx) - | Name(identifier id, expr_context ctx) - | List(expr* elts, expr_context ctx) - | Tuple(expr* elts, expr_context ctx) - - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - expr_context = Load | Store | Del | AugLoad | AugStore | Param - - slice = Slice(expr? lower, expr? upper, expr? step) - | ExtSlice(slice* dims) - | Index(expr value) - - boolop = And | Or - - operator = Add | Sub | Mult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub - - cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn - - comprehension = (expr target, expr iter, expr* ifs) - - -- not sure what to call the first argument for raise and except - excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) - attributes (int lineno, int col_offset) - - arguments = (arg* args, identifier? vararg, expr? varargannotation, - arg* kwonlyargs, identifier? kwarg, - expr? kwargannotation, expr* defaults, - expr* kw_defaults) - arg = (identifier arg, expr? annotation) - - -- keyword arguments supplied to call - keyword = (identifier arg, expr value) - - -- import name with optional 'as' alias. - alias = (identifier name, identifier? asname) + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr *decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + | Raise(expr? exc, expr? cause) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(string s) + | Ellipsis + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, identifier? vararg, expr? varargannotation, + arg* kwonlyargs, identifier? kwarg, + expr? kwargannotation, expr* defaults, + expr* kw_defaults) + arg = (identifier arg, expr? annotation) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) } diff --git a/docs/upgrade/ast/python3_2.ast b/docs/upgrade/ast/python3_2.ast index ab325d9..054d63e 100644 --- a/docs/upgrade/ast/python3_2.ast +++ b/docs/upgrade/ast/python3_2.ast @@ -3,142 +3,144 @@ module Python version "3.2" { - mod = Module(stmt* body) - | Interactive(stmt* body) - | Expression(expr body) - - -- not really an actual node but useful in Jython's typesystem. - | Suite(stmt* body) - - stmt = FunctionDef(identifier name, - arguments args, - stmt* body, - expr* decorator_list, - expr? returns) - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - expr? starargs, - expr? kwargs, - stmt* body, - expr* decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(expr context_expr, expr? optional_vars, stmt* body) - - | Raise(expr? exc, expr? cause) - | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) - | TryFinally(stmt* body, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? - expr = BoolOp(boolop op, expr* values) - | BinOp(expr left, operator op, expr right) - | UnaryOp(unaryop op, expr operand) - | Lambda(arguments args, expr body) - | IfExp(expr test, expr body, expr orelse) - | Dict(expr* keys, expr* values) - | Set(expr* elts) - | ListComp(expr elt, comprehension* generators) - | SetComp(expr elt, comprehension* generators) - | DictComp(expr key, expr value, comprehension* generators) - | GeneratorExp(expr elt, comprehension* generators) - -- the grammar constrains where yield expressions can occur - | Yield(expr? value) - -- need sequences for compare to distinguish between - -- x < 4 < 3 and (x < 4) < 3 - | Compare(expr left, cmpop* ops, expr* comparators) - | Call(expr func, expr* args, keyword* keywords, - expr? starargs, expr? kwargs) - | Num(object n) -- a number as a PyObject. - | Str(string s) -- need to specify raw, unicode, etc? - | Bytes(string s) - | Ellipsis - -- other literals? bools? - - -- the following expression can appear in assignment context - | Attribute(expr value, identifier attr, expr_context ctx) - | Subscript(expr value, slice slice, expr_context ctx) - | Starred(expr value, expr_context ctx) - | Name(identifier id, expr_context ctx) - | List(expr* elts, expr_context ctx) - | Tuple(expr* elts, expr_context ctx) - - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - expr_context = Load | Store | Del | AugLoad | AugStore | Param - - slice = Slice(expr? lower, expr? upper, expr? step) - | ExtSlice(slice* dims) - | Index(expr value) - - boolop = And | Or - - operator = Add - | Sub - | Mult - | Div - | Mod - | Pow - | LShift - | RShift - | BitOr - | BitXor - | BitAnd - | FloorDiv - - unaryop = Invert - | Not - | UAdd - | USub - - cmpop = Eq - | NotEq - | Lt - | LtE - | Gt - | GtE - | Is - | IsNot - | In - | NotIn - - comprehension = (expr target, expr iter, expr* ifs) - - -- not sure what to call the first argument for raise and except - excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) - attributes (int lineno, int col_offset) - - arguments = (arg* args, identifier? vararg, expr? varargannotation, - arg* kwonlyargs, identifier? kwarg, - expr? kwargannotation, expr* defaults, - expr* kw_defaults) - arg = (identifier arg, expr? annotation) - - -- keyword arguments supplied to call - keyword = (identifier arg, expr value) - - -- import name with optional 'as' alias. - alias = (identifier name, identifier? asname) + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + | Raise(expr? exc, expr? cause) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(string s) + | Ellipsis + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load | Store | Del | AugLoad | AugStore | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, identifier? vararg, expr? varargannotation, + arg* kwonlyargs, identifier? kwarg, + expr? kwargannotation, expr* defaults, + expr* kw_defaults) + arg = (identifier arg, expr? annotation) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) } diff --git a/docs/upgrade/ast/python3_3.ast b/docs/upgrade/ast/python3_3.ast index 30c19b5..ad9e258 100644 --- a/docs/upgrade/ast/python3_3.ast +++ b/docs/upgrade/ast/python3_3.ast @@ -15,42 +15,42 @@ module Python version "3.3" stmt* body, expr* decorator_list, expr? returns) - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - expr? starargs, - expr? kwargs, - stmt* body, - expr* decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(withitem* items, stmt* body) - - | Raise(expr? exc, expr? cause) - | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? expr = BoolOp(boolop op, expr* values) | BinOp(expr left, operator op, expr right) | UnaryOp(unaryop op, expr operand) @@ -87,18 +87,37 @@ module Python version "3.3" -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset) - expr_context = Load | Store | Del | AugLoad | AugStore | Param + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param slice = Slice(expr? lower, expr? upper, expr? step) | ExtSlice(slice* dims) | Index(expr value) - boolop = And | Or - - operator = Add | Sub | Mult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub cmpop = Eq | NotEq diff --git a/docs/upgrade/ast/python3_4.ast b/docs/upgrade/ast/python3_4.ast index e2bc06f..edfb756 100644 --- a/docs/upgrade/ast/python3_4.ast +++ b/docs/upgrade/ast/python3_4.ast @@ -15,36 +15,38 @@ module Python version "3.4" stmt* body, expr* decorator_list, expr? returns) - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - expr? starargs, - expr? kwargs, - stmt* body, - expr* decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(withitem* items, stmt* body) - - | Raise(expr? exc, expr? cause) - | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue -- XXX Jython will be different -- col_offset is the byte offset in the utf8 string the parser uses @@ -69,7 +71,7 @@ module Python version "3.4" -- x < 4 < 3 and (x < 4) < 3 | Compare(expr left, cmpop* ops, expr* comparators) | Call(expr func, expr* args, keyword* keywords, - expr? starargs, expr? kwargs) + expr? starargs, expr? kwargs) | Num(object n) -- a number as a PyObject. | Str(string s) -- need to specify raw, unicode, etc? | Bytes(bytes s) @@ -98,7 +100,8 @@ module Python version "3.4" | ExtSlice(slice* dims) | Index(expr value) - boolop = And | Or + boolop = And + | Or operator = Add | Sub diff --git a/docs/upgrade/ast/python3_5.ast b/docs/upgrade/ast/python3_5.ast index d43061d..dfe5bf1 100644 --- a/docs/upgrade/ast/python3_5.ast +++ b/docs/upgrade/ast/python3_5.ast @@ -12,45 +12,45 @@ module Python version "3.5" stmt = FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns) - | AsyncFunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list, expr? returns) - - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - stmt* body, - expr* decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(withitem* items, stmt* body) - | AsyncWith(withitem* items, stmt* body) - - | Raise(expr? exc, expr? cause) - | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? + | AsyncFunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) + + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + | AsyncWith(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? expr = BoolOp(boolop op, expr* values) | BinOp(expr left, operator op, expr right) | UnaryOp(unaryop op, expr operand) @@ -87,20 +87,49 @@ module Python version "3.5" -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset) - expr_context = Load | Store | Del | AugLoad | AugStore | Param + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param slice = Slice(expr? lower, expr? upper, expr? step) | ExtSlice(slice* dims) | Index(expr value) - boolop = And | Or - - operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub - - cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn + boolop = And + | Or + + operator = Add + | Sub + | Mult + | MatMult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn comprehension = (expr target, expr iter, expr* ifs) diff --git a/docs/upgrade/ast/python3_6.ast b/docs/upgrade/ast/python3_6.ast index d829426..6cf106a 100644 --- a/docs/upgrade/ast/python3_6.ast +++ b/docs/upgrade/ast/python3_6.ast @@ -19,50 +19,52 @@ module Python version "3.6" stmt* body, expr* decorator_list, expr? returns) - | AsyncFunctionDef(identifier name, - arguments args, - stmt* body, - expr* decorator_list, - expr? returns) - - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - stmt* body, - expr* decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - -- 'simple' indicates that we annotate simple name without parens - | AnnAssign(expr target, expr annotation, expr? value, int simple) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(withitem* items, stmt* body) - | AsyncWith(withitem* items, stmt* body) - - | Raise(expr? exc, expr? cause) - | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? + | AsyncFunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns) + + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + -- 'simple' indicates that we annotate simple name without parens + | AnnAssign(expr target, expr annotation, expr? value, int simple) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + | AsyncWith(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? expr = BoolOp(boolop op, expr* values) | BinOp(expr left, operator op, expr right) | UnaryOp(unaryop op, expr operand) @@ -113,7 +115,8 @@ module Python version "3.6" | ExtSlice(slice* dims) | Index(expr value) - boolop = And | Or + boolop = And + | Or operator = Add | Sub From 99080ebcd1943883be5ff92a667e4001c798b0df Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 4 Feb 2017 15:19:41 +0100 Subject: [PATCH 03/16] roadmap and who to contribut started --- docs/contributing/index.rst | 8 +++++++- docs/roadmap/index.rst | 25 ++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index da33d08..2d1d759 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -1,4 +1,10 @@ Contributing ============ -https://trello.com/b/pKaXJIlT/restrictedpython +Contributing to RestrictedPython 4+ + + +* `Trello Board`_ + + +.. _`Trello Board`: https://trello.com/b/pKaXJIlT/restrictedpython diff --git a/docs/roadmap/index.rst b/docs/roadmap/index.rst index 379b54c..7aa003e 100644 --- a/docs/roadmap/index.rst +++ b/docs/roadmap/index.rst @@ -1,4 +1,27 @@ Roadmap for RestrictedPython ============================ -https://trello.com/b/pKaXJIlT/restrictedpython +A few of the action items currently worked on is on our `Trello Board`_. + +.. _`Trello Board`: https://trello.com/b/pKaXJIlT/restrictedpython + +RestrictedPython 4.0 +-------------------- + +A feature complete rewrite of RestrictedPython using ``ast`` module instead of ``compile`` package. +RestrictedPython 4.0 should not add any new or remove restrictions. + +A detailed documentation that support usage and further development. + +Full code coverage tests. + +RestrictedPython 4.1+ +--------------------- + +Enhance RestrictedPython, declare deprecations and possible new restrictions. + +RestrictedPython 5.0+ +--------------------- + +* Python 3+ only, no more support for Python 2.7 +* mypy - Static Code Analysis Annotations From a06f0dd405587e010f1e9c61de5f3f1db2a838fb Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Fri, 3 Feb 2017 18:02:30 +0100 Subject: [PATCH 04/16] more docs --- README.rst | 7 ++ README.txt | 1 - CHANGES.rst => docs/CHANGES.rst | 0 docs/contributing/index.rst | 4 + docs/index.rst | 5 + docs/install/index.rst | 10 ++ docs/roadmap/index.rst | 4 + docs/usage/api.rst | 22 +++++ docs/usage/basic_usage.rst | 81 +++++++++++++++ docs/usage/framework_usage.rst | 61 ++++++++++++ docs/usage/index.rst | 169 +------------------------------- setup.py | 6 +- 12 files changed, 201 insertions(+), 169 deletions(-) create mode 100644 README.rst delete mode 100644 README.txt rename CHANGES.rst => docs/CHANGES.rst (100%) create mode 100644 docs/contributing/index.rst create mode 100644 docs/install/index.rst create mode 100644 docs/roadmap/index.rst create mode 100644 docs/usage/api.rst create mode 100644 docs/usage/basic_usage.rst create mode 100644 docs/usage/framework_usage.rst diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..7894bcc --- /dev/null +++ b/README.rst @@ -0,0 +1,7 @@ +================ +RestrictedPython +================ + +RestrictedPython is a defined subset of the Python language which allows to provide a program input into a trusted environment. + +For full documentation please see docs/index. diff --git a/README.txt b/README.txt deleted file mode 100644 index 03610af..0000000 --- a/README.txt +++ /dev/null @@ -1 +0,0 @@ -Please refer to src/RestrictedPython/README.txt. diff --git a/CHANGES.rst b/docs/CHANGES.rst similarity index 100% rename from CHANGES.rst rename to docs/CHANGES.rst diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst new file mode 100644 index 0000000..da33d08 --- /dev/null +++ b/docs/contributing/index.rst @@ -0,0 +1,4 @@ +Contributing +============ + +https://trello.com/b/pKaXJIlT/restrictedpython diff --git a/docs/index.rst b/docs/index.rst index 253d047..2459200 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,6 +25,11 @@ Contents upgrade/index upgrade_dependencies/index + roadmap/index + contributing/index + + CHANGES + Indices and tables ================== diff --git a/docs/install/index.rst b/docs/install/index.rst new file mode 100644 index 0000000..0286941 --- /dev/null +++ b/docs/install/index.rst @@ -0,0 +1,10 @@ +Install / Depend on RestrictedPython +==================================== + +RestrictedPython is usually not used stand alone, if you use it in context of your package add it to ``install_requires`` in your ``setup.py`` or a ``requirement.txt`` used by ``pip``. + +For a standalone usage: + +.. code:: bash + + pip install RestrictedPython diff --git a/docs/roadmap/index.rst b/docs/roadmap/index.rst new file mode 100644 index 0000000..379b54c --- /dev/null +++ b/docs/roadmap/index.rst @@ -0,0 +1,4 @@ +Roadmap for RestrictedPython +============================ + +https://trello.com/b/pKaXJIlT/restrictedpython diff --git a/docs/usage/api.rst b/docs/usage/api.rst new file mode 100644 index 0000000..90195bc --- /dev/null +++ b/docs/usage/api.rst @@ -0,0 +1,22 @@ +API overview +------------ + +RestrictedPython has tree major scopes: + +1. ``compile_restricted`` methods: + + * ``compile_restricted`` + * ``compile_restricted_exec`` + * ``compile_restricted_eval`` + * ``compile_restricted_single`` + * ``compile_restricted_function`` + +2. restricted builtins + + * ``safe_builtins`` + * ``limited_builtins`` + * ``utility_builtins`` + +3. helper modules + + * ``PrintCollector`` diff --git a/docs/usage/basic_usage.rst b/docs/usage/basic_usage.rst new file mode 100644 index 0000000..522c23f --- /dev/null +++ b/docs/usage/basic_usage.rst @@ -0,0 +1,81 @@ +Basic usage +----------- + +The general workflow to execute Python code that is loaded within a Python program is: + +.. code:: Python + + source_code = """ + def do_something(): + pass + """ + + byte_code = compile(source_code, filename='', mode='exec') + exec(byte_code) + do_something() + +With RestrictedPython that workflow should be as straight forward as possible: + +.. code:: Python + + from RestrictedPython import compile_restricted as compile + + source_code = """ + def do_something(): + pass + """ + + byte_code = compile(source_code, filename='', mode='exec') + exec(byte_code) + do_something() + +With that simple addition: + +.. code:: Python + + from RestrictedPython import compile_restricted as compile + +it uses a predefined policy that checks and modify the source code and checks against a restricted subset of the Python language. +The compiled source code is still executed against the full available set of library modules and methods. + +The Python :py:func:`exec` takes three parameters: + +* ``code`` which is the compiled byte code +* ``globals`` which is global dictionary +* ``locals`` which is the local dictionary + +By limiting the entries in the ``globals`` and ``locals`` dictionaries you +restrict the access to the available library modules and methods. + +Providing defined dictionaries for ``exec()`` should be used in context of RestrictedPython. + +.. code:: Python + + byte_code = + exec(byte_code, { ... }, { ... }) + +Typically there is a defined set of allowed modules, methods and constants used in that context. +RestrictedPython provides three predefined built-ins for that: + +* ``safe_builtins`` +* ``limited_builtins`` +* ``utilities_builtins`` + +So you normally end up using: + +.. code:: Python + + from RestrictedPython import ..._builtins + from RestrictedPython import compile_restricted as compile + + source_code = """""" + + try: + byte_code = compile(source_code, filename='', mode='exec') + + used_builtins = ..._builtins + { } + exec(byte_code, used_buildins, None) + except SyntaxError as e: + ... + +One common advanced usage would be to define an own restricted builtin dictionary. diff --git a/docs/usage/framework_usage.rst b/docs/usage/framework_usage.rst new file mode 100644 index 0000000..5d5e1ed --- /dev/null +++ b/docs/usage/framework_usage.rst @@ -0,0 +1,61 @@ +.. _sec_usage_frameworks: + +Usage in frameworks and Zope +---------------------------- + +One major issue with using ``compile_restricted`` directly in a framework is, that you have to use try-except statements to handle problems and it might be a bit harder to provide useful information to the user. +RestrictedPython provides four specialized compile_restricted methods: + +* ``compile_restricted_exec`` +* ``compile_restricted_eval`` +* ``compile_restricted_single`` +* ``compile_restricted_function`` + +Those four methods return a tuple with four elements: + +* ``byte_code`` object or ``None`` if ``errors`` is not empty +* ``errors`` a tuple with error messages +* ``warnings`` a list with warnings +* ``used_names`` a set / dictionary with collected used names of library calls + +Those three information "lists" could be used to provide the user with informations about the compiled source code. + +Typical uses cases for the four specialized methods: + +* ``compile_restricted_exec`` --> Python Modules or Scripts that should be used or called by the framework itself or from user calls +* ``compile_restricted_eval`` --> Templates +* ``compile_restricted_single`` +* ``compile_restricted_function`` + +Modifying the builtins is straight forward, it is just a dictionary containing access pointers to available library elements. +Modification is normally removing elements from existing builtins or adding allowed elements by copying from globals. + +For frameworks it could possibly also be useful to change handling of specific Python language elements. +For that use case RestrictedPython provide the possibility to pass an own policy. +A policy is basically a special ``NodeTransformer`` that could be instantiated with three params for ``errors``, ``warnings`` and ``used_names``, it should be a subclass of RestrictingNodeTransformer (that subclassing will maybe later be enforced). + +.. code:: Python + + OwnRestrictingNodeTransformer(errors=[], warnings=[], used_names=[]) + +One special case (defined to unblock ports of Zope Packages to Python 3) is to actually use RestrictedPython in an unrestricted mode, by providing a Null-Policy (aka ``None``). + +All ``compile_restricted*`` methods do have a optional parameter ``policy``, where a specific policy could be provided. + +.. code:: Python + + source_code = """""" + + policy = OwnRestrictingNodeTransformer + + byte_code = compile(source_code, filename='', mode='exec', policy=policy) + exec(byte_code, { ... }, { ... }) + +The Special case "unrestricted RestrictedPython" would be: + +.. code:: Python + + source_code = """""" + + byte_code = compile(source_code, filename='', mode='exec', policy=None) + exec(byte_code, globals(), None) diff --git a/docs/usage/index.rst b/docs/usage/index.rst index 5079e0d..48bd250 100644 --- a/docs/usage/index.rst +++ b/docs/usage/index.rst @@ -1,169 +1,6 @@ Usage of RestrictedPython ========================= -API overview ------------- - -RestrictedPython has tree major scopes: - -1. ``compile_restricted`` methods: - - * ``compile_restricted`` - * ``compile_restricted_exec`` - * ``compile_restricted_eval`` - * ``compile_restricted_single`` - * ``compile_restricted_function`` - -2. restricted builtins - - * ``safe_builtins`` - * ``limited_builtins`` - * ``utility_builtins`` - -3. helper modules - - * ``PrintCollector`` - -Basic usage ------------ - -The general workflow to execute Python code that is loaded within a Python program is: - -.. code:: Python - - source_code = """ - def do_something(): - pass - """ - - byte_code = compile(source_code, filename='', mode='exec') - exec(byte_code) - do_something() - -With RestrictedPython that workflow should be as straight forward as possible: - -.. code:: Python - - from RestrictedPython import compile_restricted as compile - - source_code = """ - def do_something(): - pass - """ - - byte_code = compile(source_code, filename='', mode='exec') - exec(byte_code) - do_something() - -With that simple addition: - -.. code:: Python - - from RestrictedPython import compile_restricted as compile - -it uses a predefined policy that checks and modify the source code and checks against a restricted subset of the Python language. -The compiled source code is still executed against the full available set of library modules and methods. - -The Python :py:func:`exec` takes three parameters: - -* ``code`` which is the compiled byte code -* ``globals`` which is global dictionary -* ``locals`` which is the local dictionary - -By limiting the entries in the ``globals`` and ``locals`` dictionaries you -restrict the access to the available library modules and methods. - -Providing defined dictionaries for ``exec()`` should be used in context of RestrictedPython. - -.. code:: Python - - byte_code = - exec(byte_code, { ... }, { ... }) - -Typically there is a defined set of allowed modules, methods and constants used in that context. -RestrictedPython provides three predefined built-ins for that: - -* ``safe_builtins`` -* ``limited_builtins`` -* ``utilities_builtins`` - -So you normally end up using: - -.. code:: Python - - from RestrictedPython import ..._builtins - from RestrictedPython import compile_restricted as compile - - source_code = """""" - - try: - byte_code = compile(source_code, filename='', mode='exec') - - used_builtins = ..._builtins + { } - exec(byte_code, used_buildins, None) - except SyntaxError as e: - ... - -One common advanced usage would be to define an own restricted builtin dictionary. - -.. _sec_usage_frameworks: - -Usage in frameworks and Zope ----------------------------- - -One major issue with using ``compile_restricted`` directly in a framework is, that you have to use try-except statements to handle problems and it might be a bit harder to provide useful information to the user. -RestrictedPython provides four specialized compile_restricted methods: - -* ``compile_restricted_exec`` -* ``compile_restricted_eval`` -* ``compile_restricted_single`` -* ``compile_restricted_function`` - -Those four methods return a tuple with four elements: - -* ``byte_code`` object or ``None`` if ``errors`` is not empty -* ``errors`` a tuple with error messages -* ``warnings`` a list with warnings -* ``used_names`` a set / dictionary with collected used names of library calls - -Those three information "lists" could be used to provide the user with informations about the compiled source code. - -Typical uses cases for the four specialized methods: - -* ``compile_restricted_exec`` --> Python Modules or Scripts that should be used or called by the framework itself or from user calls -* ``compile_restricted_eval`` --> Templates -* ``compile_restricted_single`` -* ``compile_restricted_function`` - -Modifying the builtins is straight forward, it is just a dictionary containing access pointers to available library elements. -Modification is normally removing elements from existing builtins or adding allowed elements by copying from globals. - -For frameworks it could possibly also be useful to change handling of specific Python language elements. -For that use case RestrictedPython provide the possibility to pass an own policy. -A policy is basically a special ``NodeTransformer`` that could be instantiated with three params for ``errors``, ``warnings`` and ``used_names``, it should be a subclass of RestrictingNodeTransformer (that subclassing will maybe later be enforced). - -.. code:: Python - - OwnRestrictingNodeTransformer(errors=[], warnings=[], used_names=[]) - -One special case (defined to unblock ports of Zope Packages to Python 3) is to actually use RestrictedPython in an unrestricted mode, by providing a Null-Policy (aka ``None``). - -All ``compile_restricted*`` methods do have a optional parameter ``policy``, where a specific policy could be provided. - -.. code:: Python - - source_code = """""" - - policy = OwnRestrictingNodeTransformer - - byte_code = compile(source_code, filename='', mode='exec', policy=policy) - exec(byte_code, { ... }, { ... }) - -The Special case "unrestricted RestrictedPython" would be: - -.. code:: Python - - source_code = """""" - - byte_code = compile(source_code, filename='', mode='exec', policy=None) - exec(byte_code, globals(), None) +.. include:: api.rst +.. include:: basic_usage.rst +.. include:: framework_usage.rst diff --git a/setup.py b/setup.py index dfe1d49..99c9d0b 100644 --- a/setup.py +++ b/setup.py @@ -29,8 +29,10 @@ def read(*rnames): license='ZPL 2.1', description='RestrictedPython provides a restricted execution ' 'environment for Python, e.g. for running untrusted code.', - long_description=(read('src', 'RestrictedPython', 'README.rst') + '\n' + - read('CHANGES.rst')), + long_description=(read('README.rst') + '\n' + + read('docs', 'install', 'index.rst') + '\n' + + read('docs', 'usage', 'basic_usage.rst') + '\n' + + read('docs', 'CHANGES.rst')), classifiers=[ 'License :: OSI Approved :: Zope Public License', 'Programming Language :: Python', From 26e5ed0c2d5d46d939cffb64024f8e71992ce471 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 4 Feb 2017 15:19:01 +0100 Subject: [PATCH 05/16] well format ast for comparision --- docs/upgrade/ast/python2_6.ast | 61 ++++++-- docs/upgrade/ast/python2_7.ast | 56 +++++-- docs/upgrade/ast/python3_0.ast | 254 ++++++++++++++++-------------- docs/upgrade/ast/python3_1.ast | 256 ++++++++++++++++-------------- docs/upgrade/ast/python3_2.ast | 278 +++++++++++++++++---------------- docs/upgrade/ast/python3_3.ast | 105 ++++++++----- docs/upgrade/ast/python3_4.ast | 67 ++++---- docs/upgrade/ast/python3_5.ast | 125 +++++++++------ docs/upgrade/ast/python3_6.ast | 93 +++++------ 9 files changed, 733 insertions(+), 562 deletions(-) diff --git a/docs/upgrade/ast/python2_6.ast b/docs/upgrade/ast/python2_6.ast index d4af5b1..3ea12f8 100644 --- a/docs/upgrade/ast/python2_6.ast +++ b/docs/upgrade/ast/python2_6.ast @@ -81,20 +81,49 @@ module Python version "2.6" -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset) - expr_context = Load | Store | Del | AugLoad | AugStore | Param - - slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step) + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Ellipsis + | Slice(expr? lower, expr? upper, expr? step) | ExtSlice(slice* dims) | Index(expr value) - boolop = And | Or - - operator = Add | Sub | Mult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub - - cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn comprehension = (expr target, expr iter, expr* ifs) @@ -103,11 +132,11 @@ module Python version "2.6" attributes (int lineno, int col_offset) arguments = (expr* args, identifier? vararg, - identifier? kwarg, expr* defaults) + identifier? kwarg, expr* defaults) - -- keyword arguments supplied to call - keyword = (identifier arg, expr value) + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) - -- import name with optional 'as' alias. - alias = (identifier name, identifier? asname) + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) } diff --git a/docs/upgrade/ast/python2_7.ast b/docs/upgrade/ast/python2_7.ast index fc3aba1..899cdc5 100644 --- a/docs/upgrade/ast/python2_7.ast +++ b/docs/upgrade/ast/python2_7.ast @@ -11,7 +11,7 @@ module Python version "2.7" | Suite(stmt* body) stmt = FunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list) + stmt* body, expr* decorator_list) | ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list) | Return(expr? value) @@ -84,20 +84,48 @@ module Python version "2.7" -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset) - expr_context = Load | Store | Del | AugLoad | AugStore | Param + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param - slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step) + slice = Ellipsis + | Slice(expr? lower, expr? upper, expr? step) | ExtSlice(slice* dims) | Index(expr value) boolop = And | Or - operator = Add | Sub | Mult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub - - cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn comprehension = (expr target, expr iter, expr* ifs) @@ -106,11 +134,11 @@ module Python version "2.7" attributes (int lineno, int col_offset) arguments = (expr* args, identifier? vararg, - identifier? kwarg, expr* defaults) + identifier? kwarg, expr* defaults) - -- keyword arguments supplied to call - keyword = (identifier arg, expr value) + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) - -- import name with optional 'as' alias. - alias = (identifier name, identifier? asname) + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) } diff --git a/docs/upgrade/ast/python3_0.ast b/docs/upgrade/ast/python3_0.ast index 2241781..44d6e18 100644 --- a/docs/upgrade/ast/python3_0.ast +++ b/docs/upgrade/ast/python3_0.ast @@ -3,117 +3,145 @@ module Python version "3.0" { - mod = Module(stmt* body) - | Interactive(stmt* body) - | Expression(expr body) - - -- not really an actual node but useful in Jython's typesystem. - | Suite(stmt* body) - - stmt = FunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list, expr? returns) - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - expr? starargs, - expr? kwargs, - stmt* body, - expr *decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(expr context_expr, expr? optional_vars, stmt* body) - - | Raise(expr? exc, expr? cause) - | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) - | TryFinally(stmt* body, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? - expr = BoolOp(boolop op, expr* values) - | BinOp(expr left, operator op, expr right) - | UnaryOp(unaryop op, expr operand) - | Lambda(arguments args, expr body) - | IfExp(expr test, expr body, expr orelse) - | Dict(expr* keys, expr* values) - | Set(expr* elts) - | ListComp(expr elt, comprehension* generators) - | SetComp(expr elt, comprehension* generators) - | DictComp(expr key, expr value, comprehension* generators) - | GeneratorExp(expr elt, comprehension* generators) - -- the grammar constrains where yield expressions can occur - | Yield(expr? value) - -- need sequences for compare to distinguish between - -- x < 4 < 3 and (x < 4) < 3 - | Compare(expr left, cmpop* ops, expr* comparators) - | Call(expr func, expr* args, keyword* keywords, - expr? starargs, expr? kwargs) - | Num(object n) -- a number as a PyObject. - | Str(string s) -- need to specify raw, unicode, etc? - | Bytes(string s) - | Ellipsis - -- other literals? bools? - - -- the following expression can appear in assignment context - | Attribute(expr value, identifier attr, expr_context ctx) - | Subscript(expr value, slice slice, expr_context ctx) - | Starred(expr value, expr_context ctx) - | Name(identifier id, expr_context ctx) - | List(expr* elts, expr_context ctx) - | Tuple(expr* elts, expr_context ctx) - - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - expr_context = Load | Store | Del | AugLoad | AugStore | Param - - slice = Slice(expr? lower, expr? upper, expr? step) - | ExtSlice(slice* dims) - | Index(expr value) - - boolop = And | Or - - operator = Add | Sub | Mult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub - - cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn - - comprehension = (expr target, expr iter, expr* ifs) - - -- not sure what to call the first argument for raise and except - excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) - attributes (int lineno, int col_offset) - - arguments = (arg* args, identifier? vararg, expr? varargannotation, - arg* kwonlyargs, identifier? kwarg, - expr? kwargannotation, expr* defaults, - expr* kw_defaults) - arg = (identifier arg, expr? annotation) - - -- keyword arguments supplied to call - keyword = (identifier arg, expr value) - - -- import name with optional 'as' alias. - alias = (identifier name, identifier? asname) + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr *decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + | Raise(expr? exc, expr? cause) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(string s) + | Ellipsis + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, identifier? vararg, expr? varargannotation, + arg* kwonlyargs, identifier? kwarg, + expr? kwargannotation, expr* defaults, + expr* kw_defaults) + arg = (identifier arg, expr? annotation) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) } diff --git a/docs/upgrade/ast/python3_1.ast b/docs/upgrade/ast/python3_1.ast index 8b9dd80..a0bbfb6 100644 --- a/docs/upgrade/ast/python3_1.ast +++ b/docs/upgrade/ast/python3_1.ast @@ -3,117 +3,147 @@ module Python version "3.1" { - mod = Module(stmt* body) - | Interactive(stmt* body) - | Expression(expr body) - - -- not really an actual node but useful in Jython's typesystem. - | Suite(stmt* body) - - stmt = FunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list, expr? returns) - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - expr? starargs, - expr? kwargs, - stmt* body, - expr *decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(expr context_expr, expr? optional_vars, stmt* body) - - | Raise(expr? exc, expr? cause) - | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) - | TryFinally(stmt* body, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? - expr = BoolOp(boolop op, expr* values) - | BinOp(expr left, operator op, expr right) - | UnaryOp(unaryop op, expr operand) - | Lambda(arguments args, expr body) - | IfExp(expr test, expr body, expr orelse) - | Dict(expr* keys, expr* values) - | Set(expr* elts) - | ListComp(expr elt, comprehension* generators) - | SetComp(expr elt, comprehension* generators) - | DictComp(expr key, expr value, comprehension* generators) - | GeneratorExp(expr elt, comprehension* generators) - -- the grammar constrains where yield expressions can occur - | Yield(expr? value) - -- need sequences for compare to distinguish between - -- x < 4 < 3 and (x < 4) < 3 - | Compare(expr left, cmpop* ops, expr* comparators) - | Call(expr func, expr* args, keyword* keywords, - expr? starargs, expr? kwargs) - | Num(object n) -- a number as a PyObject. - | Str(string s) -- need to specify raw, unicode, etc? - | Bytes(string s) - | Ellipsis - -- other literals? bools? - - -- the following expression can appear in assignment context - | Attribute(expr value, identifier attr, expr_context ctx) - | Subscript(expr value, slice slice, expr_context ctx) - | Starred(expr value, expr_context ctx) - | Name(identifier id, expr_context ctx) - | List(expr* elts, expr_context ctx) - | Tuple(expr* elts, expr_context ctx) - - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - expr_context = Load | Store | Del | AugLoad | AugStore | Param - - slice = Slice(expr? lower, expr? upper, expr? step) - | ExtSlice(slice* dims) - | Index(expr value) - - boolop = And | Or - - operator = Add | Sub | Mult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub - - cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn - - comprehension = (expr target, expr iter, expr* ifs) - - -- not sure what to call the first argument for raise and except - excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) - attributes (int lineno, int col_offset) - - arguments = (arg* args, identifier? vararg, expr? varargannotation, - arg* kwonlyargs, identifier? kwarg, - expr? kwargannotation, expr* defaults, - expr* kw_defaults) - arg = (identifier arg, expr? annotation) - - -- keyword arguments supplied to call - keyword = (identifier arg, expr value) - - -- import name with optional 'as' alias. - alias = (identifier name, identifier? asname) + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr *decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + | Raise(expr? exc, expr? cause) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(string s) + | Ellipsis + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, identifier? vararg, expr? varargannotation, + arg* kwonlyargs, identifier? kwarg, + expr? kwargannotation, expr* defaults, + expr* kw_defaults) + arg = (identifier arg, expr? annotation) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) } diff --git a/docs/upgrade/ast/python3_2.ast b/docs/upgrade/ast/python3_2.ast index ab325d9..054d63e 100644 --- a/docs/upgrade/ast/python3_2.ast +++ b/docs/upgrade/ast/python3_2.ast @@ -3,142 +3,144 @@ module Python version "3.2" { - mod = Module(stmt* body) - | Interactive(stmt* body) - | Expression(expr body) - - -- not really an actual node but useful in Jython's typesystem. - | Suite(stmt* body) - - stmt = FunctionDef(identifier name, - arguments args, - stmt* body, - expr* decorator_list, - expr? returns) - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - expr? starargs, - expr? kwargs, - stmt* body, - expr* decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(expr context_expr, expr? optional_vars, stmt* body) - - | Raise(expr? exc, expr? cause) - | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) - | TryFinally(stmt* body, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? - expr = BoolOp(boolop op, expr* values) - | BinOp(expr left, operator op, expr right) - | UnaryOp(unaryop op, expr operand) - | Lambda(arguments args, expr body) - | IfExp(expr test, expr body, expr orelse) - | Dict(expr* keys, expr* values) - | Set(expr* elts) - | ListComp(expr elt, comprehension* generators) - | SetComp(expr elt, comprehension* generators) - | DictComp(expr key, expr value, comprehension* generators) - | GeneratorExp(expr elt, comprehension* generators) - -- the grammar constrains where yield expressions can occur - | Yield(expr? value) - -- need sequences for compare to distinguish between - -- x < 4 < 3 and (x < 4) < 3 - | Compare(expr left, cmpop* ops, expr* comparators) - | Call(expr func, expr* args, keyword* keywords, - expr? starargs, expr? kwargs) - | Num(object n) -- a number as a PyObject. - | Str(string s) -- need to specify raw, unicode, etc? - | Bytes(string s) - | Ellipsis - -- other literals? bools? - - -- the following expression can appear in assignment context - | Attribute(expr value, identifier attr, expr_context ctx) - | Subscript(expr value, slice slice, expr_context ctx) - | Starred(expr value, expr_context ctx) - | Name(identifier id, expr_context ctx) - | List(expr* elts, expr_context ctx) - | Tuple(expr* elts, expr_context ctx) - - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - expr_context = Load | Store | Del | AugLoad | AugStore | Param - - slice = Slice(expr? lower, expr? upper, expr? step) - | ExtSlice(slice* dims) - | Index(expr value) - - boolop = And | Or - - operator = Add - | Sub - | Mult - | Div - | Mod - | Pow - | LShift - | RShift - | BitOr - | BitXor - | BitAnd - | FloorDiv - - unaryop = Invert - | Not - | UAdd - | USub - - cmpop = Eq - | NotEq - | Lt - | LtE - | Gt - | GtE - | Is - | IsNot - | In - | NotIn - - comprehension = (expr target, expr iter, expr* ifs) - - -- not sure what to call the first argument for raise and except - excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) - attributes (int lineno, int col_offset) - - arguments = (arg* args, identifier? vararg, expr? varargannotation, - arg* kwonlyargs, identifier? kwarg, - expr? kwargannotation, expr* defaults, - expr* kw_defaults) - arg = (identifier arg, expr? annotation) - - -- keyword arguments supplied to call - keyword = (identifier arg, expr value) - - -- import name with optional 'as' alias. - alias = (identifier name, identifier? asname) + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + | Raise(expr? exc, expr? cause) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(string s) + | Ellipsis + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load | Store | Del | AugLoad | AugStore | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, identifier? vararg, expr? varargannotation, + arg* kwonlyargs, identifier? kwarg, + expr? kwargannotation, expr* defaults, + expr* kw_defaults) + arg = (identifier arg, expr? annotation) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) } diff --git a/docs/upgrade/ast/python3_3.ast b/docs/upgrade/ast/python3_3.ast index 30c19b5..ad9e258 100644 --- a/docs/upgrade/ast/python3_3.ast +++ b/docs/upgrade/ast/python3_3.ast @@ -15,42 +15,42 @@ module Python version "3.3" stmt* body, expr* decorator_list, expr? returns) - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - expr? starargs, - expr? kwargs, - stmt* body, - expr* decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(withitem* items, stmt* body) - - | Raise(expr? exc, expr? cause) - | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? expr = BoolOp(boolop op, expr* values) | BinOp(expr left, operator op, expr right) | UnaryOp(unaryop op, expr operand) @@ -87,18 +87,37 @@ module Python version "3.3" -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset) - expr_context = Load | Store | Del | AugLoad | AugStore | Param + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param slice = Slice(expr? lower, expr? upper, expr? step) | ExtSlice(slice* dims) | Index(expr value) - boolop = And | Or - - operator = Add | Sub | Mult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub cmpop = Eq | NotEq diff --git a/docs/upgrade/ast/python3_4.ast b/docs/upgrade/ast/python3_4.ast index e2bc06f..edfb756 100644 --- a/docs/upgrade/ast/python3_4.ast +++ b/docs/upgrade/ast/python3_4.ast @@ -15,36 +15,38 @@ module Python version "3.4" stmt* body, expr* decorator_list, expr? returns) - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - expr? starargs, - expr? kwargs, - stmt* body, - expr* decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(withitem* items, stmt* body) - - | Raise(expr? exc, expr? cause) - | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue -- XXX Jython will be different -- col_offset is the byte offset in the utf8 string the parser uses @@ -69,7 +71,7 @@ module Python version "3.4" -- x < 4 < 3 and (x < 4) < 3 | Compare(expr left, cmpop* ops, expr* comparators) | Call(expr func, expr* args, keyword* keywords, - expr? starargs, expr? kwargs) + expr? starargs, expr? kwargs) | Num(object n) -- a number as a PyObject. | Str(string s) -- need to specify raw, unicode, etc? | Bytes(bytes s) @@ -98,7 +100,8 @@ module Python version "3.4" | ExtSlice(slice* dims) | Index(expr value) - boolop = And | Or + boolop = And + | Or operator = Add | Sub diff --git a/docs/upgrade/ast/python3_5.ast b/docs/upgrade/ast/python3_5.ast index d43061d..dfe5bf1 100644 --- a/docs/upgrade/ast/python3_5.ast +++ b/docs/upgrade/ast/python3_5.ast @@ -12,45 +12,45 @@ module Python version "3.5" stmt = FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns) - | AsyncFunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list, expr? returns) - - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - stmt* body, - expr* decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(withitem* items, stmt* body) - | AsyncWith(withitem* items, stmt* body) - - | Raise(expr? exc, expr? cause) - | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? + | AsyncFunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) + + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + | AsyncWith(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? expr = BoolOp(boolop op, expr* values) | BinOp(expr left, operator op, expr right) | UnaryOp(unaryop op, expr operand) @@ -87,20 +87,49 @@ module Python version "3.5" -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset) - expr_context = Load | Store | Del | AugLoad | AugStore | Param + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param slice = Slice(expr? lower, expr? upper, expr? step) | ExtSlice(slice* dims) | Index(expr value) - boolop = And | Or - - operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift - | RShift | BitOr | BitXor | BitAnd | FloorDiv - - unaryop = Invert | Not | UAdd | USub - - cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn + boolop = And + | Or + + operator = Add + | Sub + | Mult + | MatMult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn comprehension = (expr target, expr iter, expr* ifs) diff --git a/docs/upgrade/ast/python3_6.ast b/docs/upgrade/ast/python3_6.ast index d829426..6cf106a 100644 --- a/docs/upgrade/ast/python3_6.ast +++ b/docs/upgrade/ast/python3_6.ast @@ -19,50 +19,52 @@ module Python version "3.6" stmt* body, expr* decorator_list, expr? returns) - | AsyncFunctionDef(identifier name, - arguments args, - stmt* body, - expr* decorator_list, - expr? returns) - - | ClassDef(identifier name, - expr* bases, - keyword* keywords, - stmt* body, - expr* decorator_list) - | Return(expr? value) - - | Delete(expr* targets) - | Assign(expr* targets, expr value) - | AugAssign(expr target, operator op, expr value) - -- 'simple' indicates that we annotate simple name without parens - | AnnAssign(expr target, expr annotation, expr? value, int simple) - - -- use 'orelse' because else is a keyword in target languages - | For(expr target, expr iter, stmt* body, stmt* orelse) - | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) - | While(expr test, stmt* body, stmt* orelse) - | If(expr test, stmt* body, stmt* orelse) - | With(withitem* items, stmt* body) - | AsyncWith(withitem* items, stmt* body) - - | Raise(expr? exc, expr? cause) - | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - | Assert(expr test, expr? msg) - - | Import(alias* names) - | ImportFrom(identifier? module, alias* names, int? level) - - | Global(identifier* names) - | Nonlocal(identifier* names) - | Expr(expr value) - | Pass | Break | Continue - - -- XXX Jython will be different - -- col_offset is the byte offset in the utf8 string the parser uses - attributes (int lineno, int col_offset) - - -- BoolOp() can use left & right? + | AsyncFunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns) + + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + -- 'simple' indicates that we annotate simple name without parens + | AnnAssign(expr target, expr annotation, expr? value, int simple) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + | AsyncWith(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? expr = BoolOp(boolop op, expr* values) | BinOp(expr left, operator op, expr right) | UnaryOp(unaryop op, expr operand) @@ -113,7 +115,8 @@ module Python version "3.6" | ExtSlice(slice* dims) | Index(expr value) - boolop = And | Or + boolop = And + | Or operator = Add | Sub From 4888ce92f45e6baa31cedcc53a548aeef637f7a1 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 4 Feb 2017 15:19:41 +0100 Subject: [PATCH 06/16] roadmap and who to contribut started --- docs/contributing/index.rst | 8 +++++++- docs/roadmap/index.rst | 25 ++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index da33d08..2d1d759 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -1,4 +1,10 @@ Contributing ============ -https://trello.com/b/pKaXJIlT/restrictedpython +Contributing to RestrictedPython 4+ + + +* `Trello Board`_ + + +.. _`Trello Board`: https://trello.com/b/pKaXJIlT/restrictedpython diff --git a/docs/roadmap/index.rst b/docs/roadmap/index.rst index 379b54c..7aa003e 100644 --- a/docs/roadmap/index.rst +++ b/docs/roadmap/index.rst @@ -1,4 +1,27 @@ Roadmap for RestrictedPython ============================ -https://trello.com/b/pKaXJIlT/restrictedpython +A few of the action items currently worked on is on our `Trello Board`_. + +.. _`Trello Board`: https://trello.com/b/pKaXJIlT/restrictedpython + +RestrictedPython 4.0 +-------------------- + +A feature complete rewrite of RestrictedPython using ``ast`` module instead of ``compile`` package. +RestrictedPython 4.0 should not add any new or remove restrictions. + +A detailed documentation that support usage and further development. + +Full code coverage tests. + +RestrictedPython 4.1+ +--------------------- + +Enhance RestrictedPython, declare deprecations and possible new restrictions. + +RestrictedPython 5.0+ +--------------------- + +* Python 3+ only, no more support for Python 2.7 +* mypy - Static Code Analysis Annotations From edd5682f5b299f16cd72285af8d7c0a5c8e74ab8 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 4 Feb 2017 19:32:03 +0100 Subject: [PATCH 07/16] more docs consolidation --- docs/update_notes.rst | 45 ------------------------------------------- docs/usage/index.rst | 1 + docs/usage/policy.rst | 13 +++++++++++++ 3 files changed, 14 insertions(+), 45 deletions(-) delete mode 100644 docs/update_notes.rst create mode 100644 docs/usage/policy.rst diff --git a/docs/update_notes.rst b/docs/update_notes.rst deleted file mode 100644 index 665c089..0000000 --- a/docs/update_notes.rst +++ /dev/null @@ -1,45 +0,0 @@ -Notes on the Update Process to be Python 3 compatible -===================================================== - - -Also RestrictedPython provides a way to define Policies, by redefining restricted versions of ``print``, ``getattr``, ``setattr``, ``import``, etc.. -As shortcutes it offers three stripped down versions of Pythons ``__builtins__``: - -* ``safe_builtins`` (by Guards.py) -* ``limited_builtins`` (by Limits.py), which provides restriced sequence types -* ``utilities_builtins`` (by Utilities.py), which provides access for standard modules math, random, string and for sets. - -There is also a guard function for making attributes immutable --> ``full_write_guard`` (write and delete protected) - - - -Technical foundation of RestrictedPython -........................................ - -RestrictedPython is based on the Python 2 only standard library module ``compiler`` (https://docs.python.org/2.7/library/compiler.html). -RestrictedPython based on the - -* compiler.ast -* compiler.parse -* compiler.pycodegen - -With Python 2.6 the compiler module with all its sub modules has been declared deprecated with no direct upgrade Path or recommendations for a replacement. - - -Approach --------- - -RestrictedPython is a classical approach of compiler construction to create a limited subset of an existing programming language. - -As compiler construction do have basic concepts on how to build a Programming Language and Runtime Environment. - -Defining a Programming Language means to define a regular grammar (Chomsky 3 / EBNF) first. -This grammar will be implemented in an abstract syntax tree (AST), which will be passed on to a code generator to produce a machine understandable version. - -As Python is a plattform independend programming / scripting language, this machine understandable version is a byte code which will be translated on the fly by an interpreter into machine code. -This machine code then gets executed on the specific CPU architecture, with all Operating System restriction. - -Produced byte code has to compatible with the execution environment, the Python Interpreter within this code is called. -So we must not generate the byte code that has to be returned from ``compile_restricted`` and the other ``compile_restricted_*`` methods manually, as this might harm the interpreter. -We actually don't even need that. -The Python ``compile()`` function introduced the capability to compile ``ast.AST`` code into byte code. diff --git a/docs/usage/index.rst b/docs/usage/index.rst index 48bd250..a744f6d 100644 --- a/docs/usage/index.rst +++ b/docs/usage/index.rst @@ -4,3 +4,4 @@ Usage of RestrictedPython .. include:: api.rst .. include:: basic_usage.rst .. include:: framework_usage.rst +.. include:: policy.rst diff --git a/docs/usage/policy.rst b/docs/usage/policy.rst new file mode 100644 index 0000000..7884829 --- /dev/null +++ b/docs/usage/policy.rst @@ -0,0 +1,13 @@ +Policies & builtins +------------------- + + + +Also RestrictedPython provides a way to define Policies, by redefining restricted versions of ``print``, ``getattr``, ``setattr``, ``import``, etc.. +As shortcutes it offers three stripped down versions of Pythons ``__builtins__``: + +* ``safe_builtins`` (by Guards.py) +* ``limited_builtins`` (by Limits.py), which provides restriced sequence types +* ``utilities_builtins`` (by Utilities.py), which provides access for standard modules math, random, string and for sets. + +There is also a guard function for making attributes immutable --> ``full_write_guard`` (write and delete protected) From 1aa3e308d3dfaff38d09620018c63d60cb61b992 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Tue, 21 Feb 2017 13:00:03 +0100 Subject: [PATCH 08/16] apply requested changes on docs --- README.rst | 2 +- docs/conf.py | 1 + docs/contributing/index.rst | 6 ----- docs/roadmap/index.rst | 4 --- docs/usage/basic_usage.rst | 47 +++++++++++++++++++++++++++------- docs/usage/framework_usage.rst | 37 +++++++++++++++++++------- docs/usage/policy.rst | 32 +++++++++++++++++++---- setup.py | 2 -- tox.ini | 3 ++- 9 files changed, 97 insertions(+), 37 deletions(-) diff --git a/README.rst b/README.rst index 7894bcc..14e9451 100644 --- a/README.rst +++ b/README.rst @@ -4,4 +4,4 @@ RestrictedPython RestrictedPython is a defined subset of the Python language which allows to provide a program input into a trusted environment. -For full documentation please see docs/index. +For full documentation please see http://restrictedpython.readthedocs.io/en/python3_update/ or local docs/index. diff --git a/docs/conf.py b/docs/conf.py index da3ffea..39cf1bb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,6 +31,7 @@ extensions = [ 'sphinx.ext.intersphinx', 'sphinx.ext.todo', + 'sphinx.ext.doctest', ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index 2d1d759..2b62e40 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -2,9 +2,3 @@ Contributing ============ Contributing to RestrictedPython 4+ - - -* `Trello Board`_ - - -.. _`Trello Board`: https://trello.com/b/pKaXJIlT/restrictedpython diff --git a/docs/roadmap/index.rst b/docs/roadmap/index.rst index 7aa003e..1333295 100644 --- a/docs/roadmap/index.rst +++ b/docs/roadmap/index.rst @@ -1,10 +1,6 @@ Roadmap for RestrictedPython ============================ -A few of the action items currently worked on is on our `Trello Board`_. - -.. _`Trello Board`: https://trello.com/b/pKaXJIlT/restrictedpython - RestrictedPython 4.0 -------------------- diff --git a/docs/usage/basic_usage.rst b/docs/usage/basic_usage.rst index 522c23f..6aad2e5 100644 --- a/docs/usage/basic_usage.rst +++ b/docs/usage/basic_usage.rst @@ -14,28 +14,53 @@ The general workflow to execute Python code that is loaded within a Python progr exec(byte_code) do_something() +.. doctest:: + :hide: + + >>> source_code = """ + ... def do_something(): + ... pass + ... """ + >>> byte_code = compile(source_code, filename='', mode='exec') + >>> exec(byte_code) + >>> do_something() + + With RestrictedPython that workflow should be as straight forward as possible: .. code:: Python - from RestrictedPython import compile_restricted as compile + from RestrictedPython import compile_restricted source_code = """ def do_something(): pass """ - byte_code = compile(source_code, filename='', mode='exec') + byte_code = compile_restricted(source_code, filename='', mode='exec') exec(byte_code) do_something() -With that simple addition: + +.. doctest:: + :hide: + + >>> from RestrictedPython import compile_restricted + >>> source_code = """ + ... def do_something(): + ... pass + ... """ + >>> byte_code = compile_restricted(source_code, filename='', mode='exec') + >>> exec(byte_code) + >>> do_something() + +You might also use the replacement import: .. code:: Python from RestrictedPython import compile_restricted as compile -it uses a predefined policy that checks and modify the source code and checks against a restricted subset of the Python language. +``compile_restricted`` uses a predefined policy that checks and modify the source code and checks against a restricted subset of the Python language. The compiled source code is still executed against the full available set of library modules and methods. The Python :py:func:`exec` takes three parameters: @@ -55,7 +80,7 @@ Providing defined dictionaries for ``exec()`` should be used in context of Restr exec(byte_code, { ... }, { ... }) Typically there is a defined set of allowed modules, methods and constants used in that context. -RestrictedPython provides three predefined built-ins for that: +RestrictedPython provides three predefined built-ins for that (see :ref:`predefined_builtins` for details): * ``safe_builtins`` * ``limited_builtins`` @@ -65,15 +90,19 @@ So you normally end up using: .. code:: Python - from RestrictedPython import ..._builtins - from RestrictedPython import compile_restricted as compile + #from RestrictedPython import ..._builtins + from RestrictedPython import safe_builtins + from RestrictedPython import limited_builtins + from RestrictedPython import utilities_builtins + from RestrictedPython import compile_restricted source_code = """""" try: - byte_code = compile(source_code, filename='', mode='exec') + byte_code = compile_restricted(source_code, filename='', mode='exec') - used_builtins = ..._builtins + { } + #used_builtins = ..._builtins + { } # Whitelisting additional elements + used_builtins = safe_builtins exec(byte_code, used_buildins, None) except SyntaxError as e: ... diff --git a/docs/usage/framework_usage.rst b/docs/usage/framework_usage.rst index 5d5e1ed..7c31172 100644 --- a/docs/usage/framework_usage.rst +++ b/docs/usage/framework_usage.rst @@ -11,9 +11,9 @@ RestrictedPython provides four specialized compile_restricted methods: * ``compile_restricted_single`` * ``compile_restricted_function`` -Those four methods return a tuple with four elements: +Those four methods return a named tuple (``CompileResult``) with four elements: -* ``byte_code`` object or ``None`` if ``errors`` is not empty +* ``code`` ```` object or ``None`` if ``errors`` is not empty * ``errors`` a tuple with error messages * ``warnings`` a list with warnings * ``used_names`` a set / dictionary with collected used names of library calls @@ -31,15 +31,18 @@ Modifying the builtins is straight forward, it is just a dictionary containing a Modification is normally removing elements from existing builtins or adding allowed elements by copying from globals. For frameworks it could possibly also be useful to change handling of specific Python language elements. -For that use case RestrictedPython provide the possibility to pass an own policy. -A policy is basically a special ``NodeTransformer`` that could be instantiated with three params for ``errors``, ``warnings`` and ``used_names``, it should be a subclass of RestrictingNodeTransformer (that subclassing will maybe later be enforced). +For that use case RestrictedPython provides the possibility to pass an own policy. + +A policy is basically a special ``NodeTransformer`` that could be instantiated with three params for ``errors``, ``warnings`` and ``used_names``, it should be a subclass of RestrictedPython.RestrictingNodeTransformer. + +.. todo:: + + write doctests for following code .. code:: Python OwnRestrictingNodeTransformer(errors=[], warnings=[], used_names=[]) -One special case (defined to unblock ports of Zope Packages to Python 3) is to actually use RestrictedPython in an unrestricted mode, by providing a Null-Policy (aka ``None``). - All ``compile_restricted*`` methods do have a optional parameter ``policy``, where a specific policy could be provided. .. code:: Python @@ -48,14 +51,30 @@ All ``compile_restricted*`` methods do have a optional parameter ``policy``, whe policy = OwnRestrictingNodeTransformer - byte_code = compile(source_code, filename='', mode='exec', policy=policy) + byte_code = compile_restricted(source_code, filename='', mode='exec', policy=policy) exec(byte_code, { ... }, { ... }) -The Special case "unrestricted RestrictedPython" would be: +One special case "unrestricted RestrictedPython" (defined to unblock ports of Zope Packages to Python 3) is to actually use RestrictedPython in an unrestricted mode, by providing a Null-Policy (aka ``None``). +That special case would be written as: .. code:: Python source_code = """""" - byte_code = compile(source_code, filename='', mode='exec', policy=None) + byte_code = compile_restricted(source_code, filename='', mode='exec', policy=None) exec(byte_code, globals(), None) + +.. doctest:: + :hide: + + >>> from RestrictedPython import compile_restricted + >>> + >>> source_code = """ + ... def do_something(): + ... pass + ... + ... do_something() + ... """ + >>> + >>> byte_code = compile_restricted(source_code, filename='', mode='exec', policy=None) + >>> exec(byte_code, globals(), None) diff --git a/docs/usage/policy.rst b/docs/usage/policy.rst index 7884829..bce3413 100644 --- a/docs/usage/policy.rst +++ b/docs/usage/policy.rst @@ -1,13 +1,35 @@ +.. _policy_builtins: + Policies & builtins ------------------- +.. todo:: + + Should be described in detail. + Especially the difference between builtins and a policy which is a NodeTransformer. + + +RestrictedPython provides a way to define Policies, by redefining restricted versions of ``print``, ``getattr``, ``setattr``, ``import``, etc.. +As shortcuts it offers three stripped down versions of Pythons ``__builtins__``: + +.. _predefined_builtins: + +Predefined builtins +................... + +.. todo:: + + Describe more in details + +* ``safe_builtins`` a safe set of builtin modules and functions, +* ``limited_builtins`` which provides restricted sequence types, +* ``utilities_builtins`` which provides access for standard modules math, random, string and for sets. +Guards +...... -Also RestrictedPython provides a way to define Policies, by redefining restricted versions of ``print``, ``getattr``, ``setattr``, ``import``, etc.. -As shortcutes it offers three stripped down versions of Pythons ``__builtins__``: +.. todo:: -* ``safe_builtins`` (by Guards.py) -* ``limited_builtins`` (by Limits.py), which provides restriced sequence types -* ``utilities_builtins`` (by Utilities.py), which provides access for standard modules math, random, string and for sets. + Describe Guards and predefined guard methods in details There is also a guard function for making attributes immutable --> ``full_write_guard`` (write and delete protected) diff --git a/setup.py b/setup.py index 99c9d0b..503918d 100644 --- a/setup.py +++ b/setup.py @@ -30,8 +30,6 @@ def read(*rnames): description='RestrictedPython provides a restricted execution ' 'environment for Python, e.g. for running untrusted code.', long_description=(read('README.rst') + '\n' + - read('docs', 'install', 'index.rst') + '\n' + - read('docs', 'usage', 'basic_usage.rst') + '\n' + read('docs', 'CHANGES.rst')), classifiers=[ 'License :: OSI Approved :: Zope Public License', diff --git a/tox.ini b/tox.ini index e72abc8..9079330 100644 --- a/tox.ini +++ b/tox.ini @@ -8,9 +8,9 @@ envlist = py35, py36, pypy, + docs, coverage-report, isort, - docs, skip_missing_interpreters = False [testenv] @@ -74,6 +74,7 @@ commands = flake8 --doctests src tests setup.py basepython = python2.7 commands = sphinx-build -b html -d build/docs/doctrees docs build/docs/html + sphinx-build -b doctest docs build/docs/doctrees deps = .[docs] Sphinx From 3f56aca0b6eb1aa9b823ab33451ce38e89d39abf Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Tue, 21 Feb 2017 13:13:49 +0100 Subject: [PATCH 09/16] more doc und doctests fixtures --- docs/usage/basic_usage.rst | 29 +++++++++++++++++++++++++---- docs/usage/policy.rst | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/docs/usage/basic_usage.rst b/docs/usage/basic_usage.rst index 6aad2e5..3763da5 100644 --- a/docs/usage/basic_usage.rst +++ b/docs/usage/basic_usage.rst @@ -41,7 +41,6 @@ With RestrictedPython that workflow should be as straight forward as possible: exec(byte_code) do_something() - .. doctest:: :hide: @@ -84,7 +83,7 @@ RestrictedPython provides three predefined built-ins for that (see :ref:`predefi * ``safe_builtins`` * ``limited_builtins`` -* ``utilities_builtins`` +* ``utility_builtins`` So you normally end up using: @@ -93,7 +92,7 @@ So you normally end up using: #from RestrictedPython import ..._builtins from RestrictedPython import safe_builtins from RestrictedPython import limited_builtins - from RestrictedPython import utilities_builtins + from RestrictedPython import utility_builtins from RestrictedPython import compile_restricted source_code = """""" @@ -103,8 +102,30 @@ So you normally end up using: #used_builtins = ..._builtins + { } # Whitelisting additional elements used_builtins = safe_builtins - exec(byte_code, used_buildins, None) + exec(byte_code, used_builtins, None) except SyntaxError as e: ... +.. doctest:: + :hide: + + >>> #from RestrictedPython import ..._builtins + >>> from RestrictedPython import safe_builtins + >>> from RestrictedPython import limited_builtins + >>> from RestrictedPython import utility_builtins + >>> from RestrictedPython import compile_restricted + + >>> source_code = """ + ... def do_something(): + ... pass + ... """ + + >>> try: + ... byte_code = compile_restricted(source_code, filename='', mode='exec') + ... #used_builtins = ..._builtins + { } # Whitelisting additional elements + ... used_builtins = safe_builtins + ... exec(byte_code, used_builtins, None) + ... except SyntaxError as e: + ... pass + One common advanced usage would be to define an own restricted builtin dictionary. diff --git a/docs/usage/policy.rst b/docs/usage/policy.rst index bce3413..119e1cb 100644 --- a/docs/usage/policy.rst +++ b/docs/usage/policy.rst @@ -23,7 +23,7 @@ Predefined builtins * ``safe_builtins`` a safe set of builtin modules and functions, * ``limited_builtins`` which provides restricted sequence types, -* ``utilities_builtins`` which provides access for standard modules math, random, string and for sets. +* ``utility_builtins`` which provides access for standard modules math, random, string and for sets. Guards ...... From d1ef607a3e4c434b2d926b19b506751372109f52 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Tue, 21 Feb 2017 14:53:34 +0100 Subject: [PATCH 10/16] rework doctests in documentation to use .. testcode:: instead of .. doctest:: --- docs/conf.py | 4 ++ docs/contributing/index.rst | 7 ++++ docs/usage/basic_usage.rst | 70 +++++++--------------------------- docs/usage/framework_usage.rst | 43 +++++++++------------ 4 files changed, 43 insertions(+), 81 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 39cf1bb..920934d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -118,6 +118,10 @@ } +# Options for sphinx.ext.todo: +todo_include_todos = False +todo_emit_warnings = True + # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index 2b62e40..62c3fb3 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -2,3 +2,10 @@ Contributing ============ Contributing to RestrictedPython 4+ + + + +Todos +----- + +.. todolist:: diff --git a/docs/usage/basic_usage.rst b/docs/usage/basic_usage.rst index 3763da5..bb89498 100644 --- a/docs/usage/basic_usage.rst +++ b/docs/usage/basic_usage.rst @@ -3,7 +3,7 @@ Basic usage The general workflow to execute Python code that is loaded within a Python program is: -.. code:: Python +.. testcode:: source_code = """ def do_something(): @@ -14,21 +14,9 @@ The general workflow to execute Python code that is loaded within a Python progr exec(byte_code) do_something() -.. doctest:: - :hide: - - >>> source_code = """ - ... def do_something(): - ... pass - ... """ - >>> byte_code = compile(source_code, filename='', mode='exec') - >>> exec(byte_code) - >>> do_something() - - With RestrictedPython that workflow should be as straight forward as possible: -.. code:: Python +.. testcode:: Python from RestrictedPython import compile_restricted @@ -41,21 +29,9 @@ With RestrictedPython that workflow should be as straight forward as possible: exec(byte_code) do_something() -.. doctest:: - :hide: - - >>> from RestrictedPython import compile_restricted - >>> source_code = """ - ... def do_something(): - ... pass - ... """ - >>> byte_code = compile_restricted(source_code, filename='', mode='exec') - >>> exec(byte_code) - >>> do_something() - You might also use the replacement import: -.. code:: Python +.. testcode:: Python from RestrictedPython import compile_restricted as compile @@ -87,45 +63,27 @@ RestrictedPython provides three predefined built-ins for that (see :ref:`predefi So you normally end up using: -.. code:: Python +.. testcode:: - #from RestrictedPython import ..._builtins + from RestrictedPython import compile_restricted + + # from RestrictedPython import ..._builtins from RestrictedPython import safe_builtins from RestrictedPython import limited_builtins from RestrictedPython import utility_builtins - from RestrictedPython import compile_restricted - source_code = """""" + source_code = """ + def do_something(): + pass + """ try: - byte_code = compile_restricted(source_code, filename='', mode='exec') + byte_code = compile_restricted(source_code, filename='', mode='exec') - #used_builtins = ..._builtins + { } # Whitelisting additional elements + # used_builtins = ..._builtins + { } # Whitelisting additional elements used_builtins = safe_builtins exec(byte_code, used_builtins, None) except SyntaxError as e: - ... - -.. doctest:: - :hide: - - >>> #from RestrictedPython import ..._builtins - >>> from RestrictedPython import safe_builtins - >>> from RestrictedPython import limited_builtins - >>> from RestrictedPython import utility_builtins - >>> from RestrictedPython import compile_restricted - - >>> source_code = """ - ... def do_something(): - ... pass - ... """ - - >>> try: - ... byte_code = compile_restricted(source_code, filename='', mode='exec') - ... #used_builtins = ..._builtins + { } # Whitelisting additional elements - ... used_builtins = safe_builtins - ... exec(byte_code, used_builtins, None) - ... except SyntaxError as e: - ... pass + pass One common advanced usage would be to define an own restricted builtin dictionary. diff --git a/docs/usage/framework_usage.rst b/docs/usage/framework_usage.rst index 7c31172..80d5c9f 100644 --- a/docs/usage/framework_usage.rst +++ b/docs/usage/framework_usage.rst @@ -35,46 +35,39 @@ For that use case RestrictedPython provides the possibility to pass an own polic A policy is basically a special ``NodeTransformer`` that could be instantiated with three params for ``errors``, ``warnings`` and ``used_names``, it should be a subclass of RestrictedPython.RestrictingNodeTransformer. -.. todo:: +.. testcode:: Python - write doctests for following code + from RestrictedPython import compile_restricted + from RestrictedPython import RestrictingNodeTransformer + class OwnRestrictingNodeTransformer(RestrictingNodeTransformer): + pass -.. code:: Python - - OwnRestrictingNodeTransformer(errors=[], warnings=[], used_names=[]) + policy = OwnRestrictingNodeTransformer(errors=[], warnings=[], used_names=[]) All ``compile_restricted*`` methods do have a optional parameter ``policy``, where a specific policy could be provided. -.. code:: Python +.. testcode:: Python - source_code = """""" + source_code = """ + def do_something(): + pass + """ policy = OwnRestrictingNodeTransformer byte_code = compile_restricted(source_code, filename='', mode='exec', policy=policy) - exec(byte_code, { ... }, { ... }) + # exec(byte_code, { ... }, { ... }) + exec(byte_code, globals(), None) One special case "unrestricted RestrictedPython" (defined to unblock ports of Zope Packages to Python 3) is to actually use RestrictedPython in an unrestricted mode, by providing a Null-Policy (aka ``None``). That special case would be written as: -.. code:: Python +.. testcode:: Python - source_code = """""" + source_code = """ + def do_something(): + pass + """ byte_code = compile_restricted(source_code, filename='', mode='exec', policy=None) exec(byte_code, globals(), None) - -.. doctest:: - :hide: - - >>> from RestrictedPython import compile_restricted - >>> - >>> source_code = """ - ... def do_something(): - ... pass - ... - ... do_something() - ... """ - >>> - >>> byte_code = compile_restricted(source_code, filename='', mode='exec', policy=None) - >>> exec(byte_code, globals(), None) From f5910e7c84315ca3df7508ea0b750a57b0c19e76 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Tue, 21 Feb 2017 15:02:42 +0100 Subject: [PATCH 11/16] some cleanup for readabitlity of docs --- docs/usage/basic_usage.rst | 11 ++++++++--- docs/usage/framework_usage.rst | 17 ++++++++++++++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/usage/basic_usage.rst b/docs/usage/basic_usage.rst index bb89498..8ddb2b5 100644 --- a/docs/usage/basic_usage.rst +++ b/docs/usage/basic_usage.rst @@ -25,7 +25,9 @@ With RestrictedPython that workflow should be as straight forward as possible: pass """ - byte_code = compile_restricted(source_code, filename='', mode='exec') + byte_code = compile_restricted(source_code, + filename='', + mode='exec') exec(byte_code) do_something() @@ -78,9 +80,12 @@ So you normally end up using: """ try: - byte_code = compile_restricted(source_code, filename='', mode='exec') + byte_code = compile_restricted(source_code, + filename='', + mode='exec') - # used_builtins = ..._builtins + { } # Whitelisting additional elements + # Whitelisting additional elements (modules and methods) if needed: + # used_builtins = ..._builtins + { } used_builtins = safe_builtins exec(byte_code, used_builtins, None) except SyntaxError as e: diff --git a/docs/usage/framework_usage.rst b/docs/usage/framework_usage.rst index 80d5c9f..5b635f9 100644 --- a/docs/usage/framework_usage.rst +++ b/docs/usage/framework_usage.rst @@ -39,10 +39,13 @@ A policy is basically a special ``NodeTransformer`` that could be instantiated w from RestrictedPython import compile_restricted from RestrictedPython import RestrictingNodeTransformer + class OwnRestrictingNodeTransformer(RestrictingNodeTransformer): pass - policy = OwnRestrictingNodeTransformer(errors=[], warnings=[], used_names=[]) + policy_instance = OwnRestrictingNodeTransformer(errors=[], + warnings=[], + used_names=[]) All ``compile_restricted*`` methods do have a optional parameter ``policy``, where a specific policy could be provided. @@ -55,7 +58,11 @@ All ``compile_restricted*`` methods do have a optional parameter ``policy``, whe policy = OwnRestrictingNodeTransformer - byte_code = compile_restricted(source_code, filename='', mode='exec', policy=policy) + byte_code = compile_restricted(source_code, + filename='', + mode='exec', + policy=policy # Policy Class + ) # exec(byte_code, { ... }, { ... }) exec(byte_code, globals(), None) @@ -69,5 +76,9 @@ That special case would be written as: pass """ - byte_code = compile_restricted(source_code, filename='', mode='exec', policy=None) + byte_code = compile_restricted(source_code, + filename='', + mode='exec', + policy=None # Null-Policy -> unrestricted + ) exec(byte_code, globals(), None) From 77882634f856f38eadfcc506120cbe2c5f71e950 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Tue, 21 Feb 2017 15:15:04 +0100 Subject: [PATCH 12/16] better testcode grouping --- docs/usage/basic_usage.rst | 4 ++-- docs/usage/framework_usage.rst | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/usage/basic_usage.rst b/docs/usage/basic_usage.rst index 8ddb2b5..5933a74 100644 --- a/docs/usage/basic_usage.rst +++ b/docs/usage/basic_usage.rst @@ -16,7 +16,7 @@ The general workflow to execute Python code that is loaded within a Python progr With RestrictedPython that workflow should be as straight forward as possible: -.. testcode:: Python +.. testcode:: from RestrictedPython import compile_restricted @@ -33,7 +33,7 @@ With RestrictedPython that workflow should be as straight forward as possible: You might also use the replacement import: -.. testcode:: Python +.. testcode:: from RestrictedPython import compile_restricted as compile diff --git a/docs/usage/framework_usage.rst b/docs/usage/framework_usage.rst index 5b635f9..1b341e1 100644 --- a/docs/usage/framework_usage.rst +++ b/docs/usage/framework_usage.rst @@ -35,7 +35,7 @@ For that use case RestrictedPython provides the possibility to pass an own polic A policy is basically a special ``NodeTransformer`` that could be instantiated with three params for ``errors``, ``warnings`` and ``used_names``, it should be a subclass of RestrictedPython.RestrictingNodeTransformer. -.. testcode:: Python +.. testcode:: own_policy from RestrictedPython import compile_restricted from RestrictedPython import RestrictingNodeTransformer @@ -49,7 +49,7 @@ A policy is basically a special ``NodeTransformer`` that could be instantiated w All ``compile_restricted*`` methods do have a optional parameter ``policy``, where a specific policy could be provided. -.. testcode:: Python +.. testcode:: own_policy source_code = """ def do_something(): @@ -69,7 +69,9 @@ All ``compile_restricted*`` methods do have a optional parameter ``policy``, whe One special case "unrestricted RestrictedPython" (defined to unblock ports of Zope Packages to Python 3) is to actually use RestrictedPython in an unrestricted mode, by providing a Null-Policy (aka ``None``). That special case would be written as: -.. testcode:: Python +.. testcode:: + + from RestrictedPython import compile_restricted source_code = """ def do_something(): From 2c07317675eb7c696cd241e6fc448e1d15a7a61f Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Thu, 2 Mar 2017 09:38:52 +0100 Subject: [PATCH 13/16] apply requested changes by @icemac --- docs/usage/basic_usage.rst | 6 +----- docs/usage/framework_usage.rst | 1 - docs/usage/policy.rst | 13 ++++++++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/usage/basic_usage.rst b/docs/usage/basic_usage.rst index 5933a74..5fa89e2 100644 --- a/docs/usage/basic_usage.rst +++ b/docs/usage/basic_usage.rst @@ -69,7 +69,6 @@ So you normally end up using: from RestrictedPython import compile_restricted - # from RestrictedPython import ..._builtins from RestrictedPython import safe_builtins from RestrictedPython import limited_builtins from RestrictedPython import utility_builtins @@ -84,10 +83,7 @@ So you normally end up using: filename='', mode='exec') - # Whitelisting additional elements (modules and methods) if needed: - # used_builtins = ..._builtins + { } - used_builtins = safe_builtins - exec(byte_code, used_builtins, None) + exec(byte_code, safe_builtins, None) except SyntaxError as e: pass diff --git a/docs/usage/framework_usage.rst b/docs/usage/framework_usage.rst index 1b341e1..0711d8a 100644 --- a/docs/usage/framework_usage.rst +++ b/docs/usage/framework_usage.rst @@ -63,7 +63,6 @@ All ``compile_restricted*`` methods do have a optional parameter ``policy``, whe mode='exec', policy=policy # Policy Class ) - # exec(byte_code, { ... }, { ... }) exec(byte_code, globals(), None) One special case "unrestricted RestrictedPython" (defined to unblock ports of Zope Packages to Python 3) is to actually use RestrictedPython in an unrestricted mode, by providing a Null-Policy (aka ``None``). diff --git a/docs/usage/policy.rst b/docs/usage/policy.rst index 119e1cb..245ebc7 100644 --- a/docs/usage/policy.rst +++ b/docs/usage/policy.rst @@ -32,4 +32,15 @@ Guards Describe Guards and predefined guard methods in details -There is also a guard function for making attributes immutable --> ``full_write_guard`` (write and delete protected) +RestrictedPython predefines several guarded access and manipulation methods: + +* ``guarded_setattr`` +* ``guarded_delattr`` +* ``guarded_iter_unpack_sequence`` +* ``guarded_unpack_sequence`` + +Those and additional methods rely on a helper construct ``full_write_guard``, which is intended to help implement immutable and semi mutable objects and attributes. + +.. todo:: + + Describe full_write_guard more in detail and how it works. From 93d634baf6f5bf5bfff0a024bf03315e77045974 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Thu, 2 Mar 2017 09:52:20 +0100 Subject: [PATCH 14/16] more todos and inclding @stephan-hof comment on compile params flags and dont_inherit --- docs/conf.py | 2 +- docs/roadmap/index.rst | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 920934d..10d4219 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -119,7 +119,7 @@ } # Options for sphinx.ext.todo: -todo_include_todos = False +todo_include_todos = True todo_emit_warnings = True # -- Options for HTML output ---------------------------------------------- diff --git a/docs/roadmap/index.rst b/docs/roadmap/index.rst index 1333295..06588a2 100644 --- a/docs/roadmap/index.rst +++ b/docs/roadmap/index.rst @@ -11,6 +11,21 @@ A detailed documentation that support usage and further development. Full code coverage tests. +.. todo:: + + Complete documentation of all public API elements with docstyle comments + https://www.python.org/dev/peps/pep-0257/ + http://thomas-cokelaer.info/tutorials/sphinx/docstring_python.html + +.. todo:: + + Resolve Discussion in https://github.com/zopefoundation/RestrictedPython/pull/39#issuecomment-283074699 + + compile_restricted optional params flags and dont_inherit will not work as expected with the current implementation. + + stephan-hof did propose a solution, should be discussed and if approved implemented. + + RestrictedPython 4.1+ --------------------- From d25914921a7f4c8992a026d59d324eef38e4df84 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Thu, 2 Mar 2017 10:00:24 +0100 Subject: [PATCH 15/16] add autodoc to roadmap --- docs/conf.py | 1 + docs/roadmap/index.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 10d4219..08b68eb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,6 +29,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.doctest', diff --git a/docs/roadmap/index.rst b/docs/roadmap/index.rst index 06588a2..3fd98aa 100644 --- a/docs/roadmap/index.rst +++ b/docs/roadmap/index.rst @@ -15,6 +15,7 @@ Full code coverage tests. Complete documentation of all public API elements with docstyle comments https://www.python.org/dev/peps/pep-0257/ + http://www.sphinx-doc.org/en/stable/ext/autodoc.html http://thomas-cokelaer.info/tutorials/sphinx/docstring_python.html .. todo:: From 69d7878f2b122bc7b72771c75b60a6a8146af326 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Fri, 3 Mar 2017 00:43:57 +0100 Subject: [PATCH 16/16] start using api doc styles --- docs/usage/api.rst | 96 +++++++++++++++++++++++++++++++++++++++++--- docs/usage/index.rst | 2 +- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/docs/usage/api.rst b/docs/usage/api.rst index 90195bc..086efab 100644 --- a/docs/usage/api.rst +++ b/docs/usage/api.rst @@ -5,11 +5,97 @@ RestrictedPython has tree major scopes: 1. ``compile_restricted`` methods: - * ``compile_restricted`` - * ``compile_restricted_exec`` - * ``compile_restricted_eval`` - * ``compile_restricted_single`` - * ``compile_restricted_function`` +.. py:method:: compile_restricted(source, filename, mode, flags, dont_inherit, policy) + :module: RestrictedPython + + Compiles source code into interpretable byte code. + + :param source: (required). The source code that should be compiled + :param filename: (optional). + :param mode: (optional). + :param flags: (optional). + :param dont_inherit: (optional). + :param policy: (optional). + :type source: str or unicode text + :type filename: str or unicode text + :type mode: str or unicode text + :type flags: int + :type dont_inherit: int + :type policy: RestrictingNodeTransformer class + :return: Byte Code + +.. py:method:: compile_restricted_exec(source, filename, flags, dont_inherit, policy) + :module: RestrictedPython + + Compiles source code into interpretable byte code. + + :param source: (required). The source code that should be compiled + :param filename: (optional). + :param flags: (optional). + :param dont_inherit: (optional). + :param policy: (optional). + :type source: str or unicode text + :type filename: str or unicode text + :type mode: str or unicode text + :type flags: int + :type dont_inherit: int + :type policy: RestrictingNodeTransformer class + :return: CompileResult (a namedtuple with code, errors, warnings, used_names) + +.. py:method:: compile_restricted_eval(source, filename, flags, dont_inherit, policy) + :module: RestrictedPython + + Compiles source code into interpretable byte code. + + :param source: (required). The source code that should be compiled + :param filename: (optional). + :param flags: (optional). + :param dont_inherit: (optional). + :param policy: (optional). + :type source: str or unicode text + :type filename: str or unicode text + :type mode: str or unicode text + :type flags: int + :type dont_inherit: int + :type policy: RestrictingNodeTransformer class + :return: CompileResult (a namedtuple with code, errors, warnings, used_names) + +.. py:method:: compile_restricted_single(source, filename, flags, dont_inherit, policy) + :module: RestrictedPython + + Compiles source code into interpretable byte code. + + :param source: (required). The source code that should be compiled + :param filename: (optional). + :param flags: (optional). + :param dont_inherit: (optional). + :param policy: (optional). + :type source: str or unicode text + :type filename: str or unicode text + :type mode: str or unicode text + :type flags: int + :type dont_inherit: int + :type policy: RestrictingNodeTransformer class + :return: CompileResult (a namedtuple with code, errors, warnings, used_names) + +.. py:method:: compile_restricted_function(p, body, name, filename, globalize=None) + :module: RestrictedPython + + Compiles source code into interpretable byte code. + + :param p: (required). + :param body: (required). + :param name: (required). + :param filename: (required). + :param globalize: (optional). + :type p: + :type body: + :type name: str or unicode text + :type filename: str or unicode text + :type globalize: + :return: byte code + + 2. restricted builtins diff --git a/docs/usage/index.rst b/docs/usage/index.rst index a744f6d..88b25c6 100644 --- a/docs/usage/index.rst +++ b/docs/usage/index.rst @@ -1,7 +1,7 @@ Usage of RestrictedPython ========================= -.. include:: api.rst .. include:: basic_usage.rst .. include:: framework_usage.rst .. include:: policy.rst +.. include:: api.rst