Merge branch 'Python3_update' into clean_up_tests

.gitignore

@@ -27,7 +27,5 @@ pyvenv.cfg
 /src/*.egg-info
 /var
 coverage.xml
-docs/Makefile
 docs/doctrees
 docs/html
-docs/make.bat

buildout.cfg

@@ -1,11 +1,16 @@
 [buildout]
 develop = .
 parts =
+    code-analysis
+    githook
     interpreter
     test
     pytest
-    code-analysis
-    githook
+    tox
+    sphinx
+    isort
+
+versions = versions
 
 [interpreter]
 recipe = zc.recipe.egg
@@ -23,6 +28,22 @@ eggs =
     pytest-flake8
     pytest-isort
     RestrictedPython
+    tox
+
+[tox]
+recipe = zc.recipe.egg
+eggs =
+    tox
+
+[sphinx]
+recipe = zc.recipe.egg
+eggs =
+    Sphinx
+
+[isort]
+recipe = zc.recipe.egg
+eggs =
+    isort
 
 [code-analysis]
 recipe = plone.recipe.codeanalysis[recommended]
@@ -34,5 +55,9 @@ flake8-max-complexity = 15
 [githook]
 recipe = plone.recipe.command
 command =
-    echo "\nbin/pytest" >> .git/hooks/pre-commit
+    #echo "\nbin/pytest" >> .git/hooks/pre-commit
+    echo "\nbin/tox" >> .git/hooks/pre-commit
     cat .git/hooks/pre-commit
+
+[versions]
+pycodestyle = 2.2.0

docs_de/Makefile → docs/Makefile

@@ -8,14 +8,14 @@ PAPER         =
 BUILDDIR      = ../build/docs
 
 # User-friendly check for sphinx-build
-ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
-	$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/)
-endif
+#ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+#	$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/)
+#endif
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 
@@ -55,19 +55,19 @@ clean:
 
 .PHONY: html
 html:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) . $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
 .PHONY: dirhtml
 dirhtml:
-	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) . $(BUILDDIR)/dirhtml
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 
 .PHONY: singlehtml
 singlehtml:
-	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) . $(BUILDDIR)/singlehtml
 	@echo
 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 

docs/RestrictedPython4/index.rst

@@ -0,0 +1,48 @@
+RestrictedPython 4+
+===================
+
+RestrictedPython 4 is a complete rewrite for Python 3 compatibility.
+
+Goals for a rewrite
+-------------------
+
+RestrictedPython is a core dependency for the Zope2 application server and therefore for the content management system Plone.
+The Zope & Plone community want to continue their projects and as Python 2 will reach its end-of-life by 2020, to be replaced by Python 3.
+Zope and Plone should become Python 3 compatible.
+
+One of the core features of Zope 2 and therefore Plone is the possibility to implement and modify Python scripts and templates through the web (TTW) without harming the application or server itself.
+
+
+Targeted Versions to support
+----------------------------
+
+For the RestrictedPython 4 update we aim to support only current Python
+versions (the ones that will have active `security support`_ after this update
+will be completed):
+
+* 2.7
+* 3.4
+* 3.5
+* 3.6
+* PyPy2.7
+
+.. _`security support` : https://docs.python.org/devguide/index.html#branchstatus
+
+We explicitly excluded Python 3.3 and PyPy3 (which is based on the Python 3.3 specification) as the changes in Python 3.4 are significant and the Python 3.3 is nearing the end of its supported lifetime.
+
+Dependencies
+------------
+
+The following packages / modules have hard dependencies on RestrictedPython:
+
+* AccessControl -->
+* zope.untrustedpython --> SelectCompiler
+* DocumentTemplate -->
+* Products.PageTemplates -->
+* Products.PythonScripts -->
+* Products.PluginIndexes -->
+* five.pt (wrapping some functions and protection for Chameleon) -->
+
+Additionally the folowing add ons have dependencies on RestrictedPython
+
+* None

docs_de/api/index.rst → docs/api/index.rst

@@ -5,4 +5,4 @@ API von RestrictedPython 4.0
 
 .. code:: Python
 
-    restricted_compile(source, filename, mode [, flags [, dont_inherit]])
+    compile_restricted(source, filename, mode [, flags [, dont_inherit]])

docs/conf.py

@@ -29,6 +29,7 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'sphinx.ext.intersphinx',
     'sphinx.ext.todo',
 ]
 
@@ -105,6 +106,16 @@
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
 
+# Intersphinx Mapping for Links between different Documentations
+intersphinx_mapping = {
+    'python2': ('https://docs.python.org/2', None),
+    'python2.7': ('https://docs.python.org/2.7', None),
+    'python3': ('https://docs.python.org/3', None),
+    'python34': ('https://docs.python.org/3.4', None),
+    'python35': ('https://docs.python.org/3.5', None),
+    'python36': ('https://docs.python.org/3.6', None),
+
+}
 
 # -- Options for HTML output ----------------------------------------------
 

docs/idea.rst

@@ -0,0 +1,86 @@
+The Idea behind RestrictedPython
+================================
+
+Python is a `Turing-complete`_ programming language.
+To offer a Python interface for users in web context is a potential security risk.
+Web frameworks and Content Management Systems (CMS) want to offer their users as much extensibility as possible through the web (TTW).
+This also means to have permissions to add functionallity via a Python Script.
+
+There should be additional preventive measures taken to ensure integrity of the application and the server itself, according to information security best practice and unrelated to Restricted Python.
+
+RestrictedPython defines a safe subset of the Python programming language.
+This is a common approach for securing a programming language.
+The `Ada Ravenscar profile`_ is another example of such an approach.
+
+Defining a secure subset of the language involves restricting the `EBNF`_ elements and explicitly allowing or disallowing language features.
+Much of the power of a programming language derives from its standard and contributed libraries, so any calling of these methods must also be checked and potentially restricted.
+RestricedPython generally disallows calls to any library that is not explicit whitelisted.
+
+As Python is a scripting language that is executed by an interpreter.
+Any Python code that should be executed have to be explict checked before executing a generated byte code by the interpreter.
+
+Python itself offers three methods that provide such a workflow:
+
+* ``compile()`` which compiles source code to byte code
+* ``exec`` / ``exec()`` which executes the byte code in the interpreter
+* ``eval`` / ``eval()`` which executes a byte code expression
+
+Therefore RestrictedPython offers a repacement for the python builtin function ``compile()`` (Python 2: https://docs.python.org/2/library/functions.html#compile / Python 3 https://docs.python.org/3/library/functions.html#compile).
+This method is defined as following:
+
+.. code:: Python
+
+    compile(source, filename, mode [, flags [, dont_inherit]])
+
+The definition of the ``compile()`` method has changed over time, but its relevant parameters ``source`` and ``mode`` still remain.
+
+There are three valid string values for ``mode``:
+
+* ``'exec'``
+* ``'eval'``
+* ``'single'``
+
+For RestricedPython this ``compile()`` method is replaced by:
+
+.. code:: Python
+
+    compile_restriced(source, filename, mode [, flags [, dont_inherit]])
+
+The primary parameter ``source`` has to be a ASCII or ``unicode`` string.
+Both methods either returns compiled byte code that the interpreter could execute or raise exceptions if the provided source code is invalid.
+
+As ``compile`` and ``compile_restricted`` just compile the provided source code to bytecode it is not sufficient to sandbox the environment, as all calls to libraries are still available.
+
+The two methods / Statements:
+
+* ``exec`` / ``exec()``
+* ``eval`` / ``eval()``
+
+have two parameters:
+
+* globals
+* locals
+
+which are a reference to the Python builtins.
+
+By modifing and restricting the avaliable moules, methods and constants from globals and locals we could limit the possible calls.
+
+Additionally RestrictedPython offers a way to define a policy which allows developers to protect access to attributes.
+This works by defining a restricted version of:
+
+* ``print``
+* ``getattr``
+* ``setattr``
+* ``import``
+
+Also RestrictedPython provides three predefined, limited versions of Python's own ``__builtins__``:
+
+* ``safe_builtins`` (by Guards.py)
+* ``limited_builtins`` (by Limits.py), which provides restriced sequence types
+* ``utilities_builtins`` (by Utilities.py), which provides access for standard modules math, random, string and for sets.
+
+Additional there exist guard functions to make attributes of Python objects immutable --> ``full_write_guard`` (write and delete protected)
+
+.. _Turing-complete: https://en.wikipedia.org/wiki/Turing_completeness
+.. _Ada Ravenscar Profile: https://en.wikipedia.org/wiki/Ravenscar_profile
+.. _EBNF: https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form

docs/index.rst

@@ -7,11 +7,20 @@
 Welcome to RestrictedPython's documentation!
 ============================================
 
-Contents:
+.. include:: idea.rst
+
+Contents
+========
 
 .. toctree::
    :maxdepth: 2
 
+   idea
+   basics/index
+   RestrictedPython3/index
+   RestrictedPython4/index
+   update/index
+   api/index
 
 
 Indices and tables

docs/update/index.rst

@@ -0,0 +1,65 @@
+Concept for a upgrade to Python 3
+=================================
+
+RestrictedPython is a classic approach of compiler construction to create a limited subset of an existing programming language.
+
+Defining a programming language requires a regular grammar (Chomsky 3 / EBNF) definition.
+This grammar will be implemented in an abstract syntax tree (AST), which will be passed on to a code generator to produce a machine-readable version.
+
+As Python is a platform independent programming language, this machine readable version is a byte code which will be translated on the fly by an interpreter into machine code.
+This machine code then gets executed on the specific CPU architecture, with the standard operating system restrictions.
+
+The bytecode produced must be compatible with the execution environment that the Python interpreter is running in, so we do not generate the byte code directly from ``compile_restricted`` and the other ``compile_restricted_*`` methods manually, it may not match what the interpreter expects.
+
+Thankfully, the Python ``compile()`` function introduced the capability to compile ``ast.AST`` code into byte code in Python 2.6, so we can return the platform-independent AST and keep bytecode generation delegated to the interpreter.
+
+As the ``compiler`` module was deprecated in Python 2.6 and was removed before Python 3.0 was released it has never been avaliable for any Python 3 version.
+So we need to move from ``compiler.ast`` to ``ast`` to support newer Python versions.
+
+``compiler.ast`` --> ``ast``
+----------------------------
+
+From the point of view of compiler design, the concepts of the compiler module and the ast module are similar.
+The ``compiler`` module predates several major improvements of the Python development like a generally applied style guide.
+While ``compiler`` still uses the old `CamelCase`_ Syntax (``visitNode(self, node, walker)``) the ``ast.AST`` did now use the Python common ``visit_Node(self, node)`` syntax.
+Also the names of classes have been changed, where ``compiler`` uses ``Walker`` and ``Mutator`` the corresponding elements in ``ast.AST`` are ``NodeVisitor`` and ``NodeTransformator``.
+
+
+ast module (Abstract Syntax Trees)
+---------------------------------
+
+The ast module consists of four areas:
+
+* AST (Basis of all Nodes) + all node class implementations
+* NodeVisitor and NodeTransformer (tool to consume and modify the AST)
+* Helper methods
+
+  * parse
+  * walk
+  * dump
+
+* Constants
+
+  * PyCF_ONLY_AST
+
+
+NodeVisitor & NodeTransformer
+-----------------------------
+
+A NodeVisitor is a class of a node / AST consumer, it reads the data by stepping through the tree without modifying it.
+In contrast, a NodeTransformer (which inherits from a NodeVisitor) is allowed to modify the tree and nodes.
+
+
+Links
+-----
+
+* Concept of Immutable Types and Python Example (https://en.wikipedia.org/wiki/Immutable_object#Python)
+* Python 3 Standard Library Documentation on AST module ``ast`` (https://docs.python.org/3/library/ast.html)
+
+  * AST Gramar of Python https://docs.python.org/3.5/library/ast.html#abstract-grammar https://docs.python.org/2.7/library/ast.html#abstract-grammar
+  * NodeVistiors (https://docs.python.org/3.5/library/ast.html#ast.NodeVisitor)
+  * NodeTransformer (https://docs.python.org/3.5/library/ast.html#ast.NodeTransformer)
+  * dump (https://docs.python.org/3.5/library/ast.html#ast.dump)
+
+* Indetail Documentation on the Python AST module ``ast`` (https://greentreesnakes.readthedocs.org/en/latest/)
+* Example how to Instrumenting the Python AST (``ast.AST``) (http://www.dalkescientific.com/writings/diary/archive/2010/02/22/instrumenting_the_ast.html)

docs/update_notes.rst

@@ -12,7 +12,6 @@ Idea of RestrictedPython
 RestrictedPython offers a replacement for the Python builtin function ``compile()`` (https://docs.python.org/2/library/functions.html#compile) which is defined as:
 
 .. code:: Python
-    :caption: compile()
 
     compile(source, filename, mode [, flags [, dont_inherit]])