diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..755d326 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,15 @@ +[run] +branch = True +source = RestrictedPython +omit = + # Tests are classically not part of source code + # and should not be calculated into coverage sum + # on the other hand, the coverage tools do a handy job on highlighting + # code branches and tests that that did not get executed. + # Therefore we include tests into coverage analysis for the moment. + #tests/*.py + #src/RestrictedPython/tests + #src/RestrictedPython/tests/*.py + +[report] +precision = 3 diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b35f2d6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +[*] +indent_style = space +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 + +[*.{py,cfg}] +indent_size = 4 + +[Makefile] +indent_style = tab diff --git a/.gitignore b/.gitignore index 6571a5f..09b8d55 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,34 @@ -*.egg-info -*.py? -.installed.cfg -.mr.developer.cfg -bin/ -build/ -develop/ -develop-eggs/ -dist/ -eggs/ -include/ -lib/ -parts/ +*.mo +*.py[cod] +.coverage* +pip-selfcheck.json +pyvenv.cfg +/.python-version +/*.egg-info +/.eggs/ +/.Python +/.cache +/.installed.cfg +/.mr.developer.cfg +/.project +/.pydevproject +/.tox +/bin +/build +/develop-eggs +/dist +/downloads +/eggs +/fake-eggs +/htmlcov +/report-*.html +/include +/lib +/share +/local.cfg +/parts +/src/*.egg-info +/var +coverage.xml +docs/doctrees +docs/html diff --git a/.travis.yml b/.travis.yml index 39ce6d6..d0d5ee7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,29 @@ language: python sudo: false python: - 2.7 + - 3.4 + - 3.5 + - 3.6 + - pypy-5.4 +env: + - ENVIRON=py + - ENVIRON=py27-rp3,py27-datetime,py36-datetime + - ENVIRON=isort,flake8,docs +matrix: + exclude: + - env: ENVIRON=isort,flake8,docs + - env: ENVIRON=py27-rp3,py27-datetime,py36-datetime + include: + - python: "3.6" + env: ENVIRON=py36-datetime,isort,flake8,docs + - python: "2.7" + env: ENVIRON=py27-rp3,py27-datetime install: - - python bootstrap.py - - bin/buildout + - pip install tox coveralls coverage script: - - bin/test + - tox -e $ENVIRON +after_success: + - coverage combine + - coveralls notifications: email: false diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..14e9451 --- /dev/null +++ b/README.rst @@ -0,0 +1,7 @@ +================ +RestrictedPython +================ + +RestrictedPython is a defined subset of the Python language which allows to provide a program input into a trusted environment. + +For full documentation please see http://restrictedpython.readthedocs.io/en/python3_update/ or local docs/index. diff --git a/README.txt b/README.txt deleted file mode 100644 index 03610af..0000000 --- a/README.txt +++ /dev/null @@ -1 +0,0 @@ -Please refer to src/RestrictedPython/README.txt. diff --git a/bootstrap.py b/bootstrap.py index a459921..1f59b21 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -49,8 +49,8 @@ parser.add_option("-t", "--accept-buildout-test-releases", dest='accept_buildout_test_releases', action="store_true", default=False, - help=("Normally, if you do not specify a --version, the " - "bootstrap script and buildout gets the newest " + help=("Normally, if you do not specify a --buildout-version, " + "the bootstrap script and buildout gets the newest " "*final* versions of zc.buildout and its recipes and " "extensions for you. If you use this flag, " "bootstrap and buildout will get the newest releases " diff --git a/buildout.cfg b/buildout.cfg index 0572067..6a0191e 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -1,12 +1,63 @@ [buildout] develop = . -parts = interpreter test +parts = + code-analysis + githook + interpreter + test + pytest + tox + sphinx + isort + +versions = versions [interpreter] recipe = zc.recipe.egg -interpreter = python -eggs = RestrictedPython +interpreter = tpython +eggs = RestrictedPython[test,develop,docs] [test] recipe = zc.recipe.testrunner eggs = RestrictedPython + +[pytest] +recipe = zc.recipe.egg +eggs = + pytest + pytest-flake8 + pytest-isort + RestrictedPython + tox + +[tox] +recipe = zc.recipe.egg +eggs = + tox + +[sphinx] +recipe = zc.recipe.egg +eggs = + Sphinx + +[isort] +recipe = zc.recipe.egg +eggs = + isort + +[code-analysis] +recipe = plone.recipe.codeanalysis[recommended] +directory = ${buildout:directory}/ +flake8 = False +flake8-exclude = bootstrap.py,bootstrap-buildout.py,docs,*.egg.,omelette +flake8-max-complexity = 15 + +[githook] +recipe = plone.recipe.command +command = + #echo "\nbin/pytest" >> .git/hooks/pre-commit + echo "\nbin/tox" >> .git/hooks/pre-commit + cat .git/hooks/pre-commit + +[versions] +pycodestyle = 2.2.0 diff --git a/CHANGES.txt b/docs/CHANGES.rst similarity index 87% rename from CHANGES.txt rename to docs/CHANGES.rst index 2049728..96a7152 100644 --- a/CHANGES.txt +++ b/docs/CHANGES.rst @@ -1,10 +1,17 @@ Changes ======= -3.7.0a1 (unreleased) --------------------- +4.0.0 (unreleased) +------------------ + +- Mostly complete rewrite based on Python AST module. + [loechel (Alexander Loechel), icemac (Michael Howitz), stephan-hof (Stephan Hofmockel), tlotze (Thomas Lotze)] + +- switch to pytest + +- The ``compile_restricted*`` functions now return a + ``namedtuple CompileResult`` instead of a simple ``tuple``. -- TBD 3.6.0 (2010-07-09) ------------------ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..9cb365c --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,230 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = ../build/docs + +# User-friendly check for sphinx-build +#ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +# $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) +#endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) . $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) . $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) . $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/RestrictedPython.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/RestrictedPython.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/RestrictedPython" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/RestrictedPython" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/docs/RestrictedPython3/index.rst b/docs/RestrictedPython3/index.rst new file mode 100644 index 0000000..21f37ef --- /dev/null +++ b/docs/RestrictedPython3/index.rst @@ -0,0 +1,41 @@ +RestrictedPython 3.6.x and before +================================= + + +Technical foundation of RestrictedPython +........................................ + +RestrictedPython is based on the Python 2 only standard library module ``compiler`` (https://docs.python.org/2.7/library/compiler.html). +RestrictedPython based on the + +* ``compiler.ast`` +* ``compiler.parse`` +* ``compiler.pycodegen`` + +With Python 2.6 the compiler module with all its sub modules has been declared deprecated with no direct upgrade Path or recommendations for a replacement. + + +Version Support of RestrictedPython 3.6.x +......................................... + +RestrictedPython 3.6.x aims on supporting Python versions: + +* 2.0 +* 2.1 +* 2.2 +* 2.3 +* 2.4 +* 2.5 +* 2.6 +* 2.7 + +Even if the README claims that Compatibility Support is form Python 2.3 - 2.7 I found some Code in RestrictedPython and related Packages which test if Python 1 is used. + +Due to this approach to support all Python 2 Versions the code uses only statements that are compatible with all of those versions. + +So old style classes and new style classes are mixed, + +The following language elements are statements and not functions: + +* exec +* print diff --git a/docs/RestrictedPython4/index.rst b/docs/RestrictedPython4/index.rst new file mode 100644 index 0000000..60b5810 --- /dev/null +++ b/docs/RestrictedPython4/index.rst @@ -0,0 +1,92 @@ +RestrictedPython 4+ +=================== + +RestrictedPython 4 is a complete rewrite for Python 3 compatibility. + +Goals for a rewrite +------------------- + +RestrictedPython is a core dependency for the Zope2 application server and therefore for the content management system Plone. +The Zope & Plone community want to continue their projects and as Python 2 will reach its end-of-life by 2020, to be replaced by Python 3. +Zope and Plone should become Python 3 compatible. + +One of the core features of Zope 2 and therefore Plone is the possibility to implement and modify Python scripts and templates through the web (TTW) without harming the application or server itself. + +As Python is a `Turing complete`_ programming language programmers don't have any limitation and could potentially harm the Application and Server itself. + +RestrictedPython and AccessControl aims on this topic to provide a reduced subset of the Python Programming language, where all functions that could harm the system are permitted by default. + +Targeted Versions to support +---------------------------- + +For the RestrictedPython 4 update we aim to support only current Python +versions (the ones that will have active `security support`_ after this update +will be completed): + +* 2.7 +* 3.4 +* 3.5 +* 3.6 +* PyPy2.7 + +.. _`security support` : https://docs.python.org/devguide/index.html#branchstatus +.. _`Turing complete`: https://en.wikipedia.org/wiki/Turing_completeness + +We explicitly excluded Python 3.3 and PyPy3 (which is based on the Python 3.3 specification) as the changes in Python 3.4 are significant and the Python 3.3 is nearing the end of its supported lifetime. + +Dependencies +------------ + +The following packages / modules have hard dependencies on RestrictedPython: + +* AccessControl --> +* zope.untrustedpython --> SelectCompiler +* DocumentTemplate --> +* Products.PageTemplates --> +* Products.PythonScripts --> +* Products.PluginIndexes --> +* five.pt (wrapping some functions and protection for Chameleon) --> + +Additionally the following add ons have dependencies on RestrictedPython + +* None + +How RestrictedPython 4+ works internally +---------------------------------------- + +RestrictedPython's core functions are split over several files: + +* __init__.py --> It exports the API directly in the ``RestrictedPython`` namespace. It should be not necessary to import from any other module inside the package. +* compile.py --> It contains the ``compile_restricted`` functions where internally ``_compile_restricted_mode`` is the important one +* transformer.py --> Home of the ``RestrictingNodeTransformer`` + +``RestrictingNodeTransformer`` +.............................. + +The ``RestrictingNodeTransformer`` is one of the core elements of RestrictedPython, it provides the base policy used by itself. + +``RestrictingNodeTransformer`` is a subclass of a ``NodeTransformer`` which has as set of ``visit_`` methods and a ``generic_visit`` method. + +``generic_visit`` is a predefined method of any ``NodeVisitor`` which sequential visit all sub nodes, in RestrictedPython this behavior is overwritten to always call a new internal method ``not_allowed(node)``. +This results in a implicit whitelisting of all allowed AST elements. +Any possible new introduced AST element in Python (new language element) will implicit be blocked and not allowed in RestrictedPython. + +So if new elements should be introduced an explicit ``visit_`` is necessary. + + +``_compile_restricted_mode`` +............................ + +``_compile_restricted_mode`` is an internal method that does the whole mapping against the used policy and compiles provided source code, with respecting the mode. +It is wrapped by the explicit functions: + +* ``compile_restricted_exec`` +* ``compile_restricted_eval`` +* ``compile_restricted_single`` +* ``compile_restricted_function`` + +They are still exposed as those are the nominal used API. + +For advanced usage this function is interesting as it is the point where the policy came into play. +If ``policy`` is ``None`` it just call the Python builtin ``compile`` method. +Else it parse the provided Python source code into an ``ast.AST`` and let it check and transform by the provided policy. diff --git a/docs/api/index.rst b/docs/api/index.rst new file mode 100644 index 0000000..d0a5d2d --- /dev/null +++ b/docs/api/index.rst @@ -0,0 +1,24 @@ +API of RestrictedPython 4.0 +=========================== + +.. code:: Python + + compile_restricted(source, filename, mode [, flags [, dont_inherit]]) + +.. code:: Python + + compile_restricted_exec(source, filename, mode [, flags [, dont_inherit [, policy]]]) + +.. code:: Python + + compile_restricted_eval(source, filename, mode [, flags [, dont_inherit [, policy]]]) + + +.. code:: Python + + compile_restricted_single(source, filename, mode [, flags [, dont_inherit [, policy]]]) + + +.. code:: Python + + compile_restricted_function(source, filename, mode [, flags [, dont_inherit [, policy]]]) diff --git a/docs/basics/index.rst b/docs/basics/index.rst new file mode 100644 index 0000000..5aaff75 --- /dev/null +++ b/docs/basics/index.rst @@ -0,0 +1,54 @@ +Grundlagen von RestrictedPython und der Sicherheitskonzepte von Zope2 +===================================================================== + + +Motivation für RestrictedPython +------------------------------- + +Python ist eine moderne und heute sehr beliebte Programmiersprache. +Viele Bereiche nutzen heute Python ganz selbstverständlich. +Waren am Anfang gerade Systemadministratoren die via Python-Skripte ihre Systeme pflegten, ist heute die PyData Community eine der größten Nutzergruppen. +Auch wird Python gerne als Lehrsprache verwendet. + +Ein Nutzungsbereich von Python unterscheidet sich fundamental: *Python-Web* bzw. *Applikations-Server die Fremdcode aufnehmen*. +Zope gehörte zu den ersten großen und erfolgreichen Python-Web-Projekten und hat sich mit als erster um dieses Thema gekümmert. + +Während in der klassischen Software-Entwicklung aber auch in der Modelierung und Analyse von Daten drei Aspekte relevant sind: + +* Verständlichkeit des Programms (--> Siehe PEP 20 "The Zen of Python" https://www.python.org/dev/peps/pep-0020/) +* die Effizienz der Programmiersprache und der Ausführungsumgebung +* Verfügbarkeit der Ausführungsumgebung + +ist ein grundlegender Aspekt, die Mächtigkeit der Programmiersprache, selten von Relevanz. +Dies liegt auch daran, dass alle gängigen Programmiersprachen die gleiche Mächtigkeit besitzten: Turing-Vollständig. +Die Theoretische Informatik kennt mehrere Stufen der Mächtigkeit einer Programmiersprache, diese bilden die Grundlage der Berechenbarkeitstheorie. +Für klassische Software-Entwicklung ist eine Turing-vollständige Programmiersprache entsprechend die richtige Wahl. + +In der klassischen Software-Welt gelten in der Regel folgende Bedingungen: + +* man bekommt eine fertige Software und führt diese aus (Beispiel: Betriebssysteme, Anwendungen und Frameworks) +* man schreibt eine Software / Skript +* man verarbeitet Daten zur Berechung und Visualisierung, ohne ein vollumfängliches Programm zu entwickeln (Beispiel: MatLab, Jupyter-Notebooks) + +Da hierbei erstmal keine Unterscheidung zwischen Open Source und Closed Source Software gemacht werden soll, da die relevante Frage eher eine Frage des Vertrauen ist. + +Die zentrale Frage ist: + + Vertraue ich der Software, bzw. den Entwicklern der Software und führe diese aus. + + + + +Python ist eine Turing-vollständige Prgrammiersprache. +Somit haben Entwickler grundsätzlich erstmal keine Limitierungen beim programmieren. + + + +und können somit potentiell die Applikation und den Server selber schaden. + +RestrictedPython und AccessControl zielen auf diese Besonderheit und versuchen einen reduzierten Subset der Programmiersprache Python zur verfügung zu stellen. +Hierzu werden erstmal alle Funktionen die potentiel das System schaden können verboten. +Genauer gesagt muss jede Funktion, egal ob eine der Python ``__builtin__``-Funktionen, der Python Standard-Library oder beliebiger Zusatz-Modulen / (Python-Eggs) explizit freigegeben werden. +Wie sprechen hier von White-Listing. + +Damit dies funktioniert, muss neben der ``restricted_compile``-Funktion auch eine API für die explizite Freigabe von Modulen und Funktionen existieren. diff --git a/docs/call.txt b/docs/call.txt new file mode 100644 index 0000000..b333f5e --- /dev/null +++ b/docs/call.txt @@ -0,0 +1,6 @@ +collective.themefragments: /collective/themefragments/traversal.py:6:from RestrictedPython import compile_restricted_function +collective.themefragments: /collective/themefragments/traversal.py:29: r = compile_restricted_function(p, body, name, filename, globalize) +Products.PythonScripts: /Products/PythonScripts/PythonScript.py:45:from RestrictedPython import compile_restricted_function +Products.PythonScripts: /Products/PythonScripts/PythonScript.py:239: return compile_restricted_function(*args, **kw) +Zope2: /Products/PageTemplates/ZRPythonExpr.py:20:from RestrictedPython import compile_restricted_eval +Zope2: /Products/PageTemplates/ZRPythonExpr.py:36: code, err, warn, use = compile_restricted_eval(text, diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..3f5aa55 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- +# +# RestrictedPython documentation build configuration file, created by +# sphinx-quickstart on Thu May 19 12:43:20 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.doctest', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'RestrictedPython' +copyright = u'2017, Zope Foundation and Contributors' +author = u'Alexander Loechel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'4.0.0.dev0' +# The full version, including alpha/beta/rc tags. +release = u'4.0.0.dev0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# Intersphinx Mapping for Links between different Documentations +intersphinx_mapping = { + 'python2': ('https://docs.python.org/2', None), + 'python2.7': ('https://docs.python.org/2.7', None), + 'python3': ('https://docs.python.org/3', None), + 'python34': ('https://docs.python.org/3.4', None), + 'python35': ('https://docs.python.org/3.5', None), + 'python36': ('https://docs.python.org/3.6', None), + +} + +# Options for sphinx.ext.todo: +todo_include_todos = True +todo_emit_warnings = True + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. +# " v documentation" by default. +#html_title = u'RestrictedPython v4.0.0.a1' + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +#html_last_updated_fmt = None + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'RestrictedPythondoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + #'preamble': '', + + # Latex figure (float) alignment + #'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'RestrictedPython.tex', u'RestrictedPython Documentation', + u'Alexander Loechel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'restrictedpython', u'RestrictedPython Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'RestrictedPython', u'RestrictedPython Documentation', + author, 'RestrictedPython', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst new file mode 100644 index 0000000..62c3fb3 --- /dev/null +++ b/docs/contributing/index.rst @@ -0,0 +1,11 @@ +Contributing +============ + +Contributing to RestrictedPython 4+ + + + +Todos +----- + +.. todolist:: diff --git a/docs/dep.txt b/docs/dep.txt new file mode 100644 index 0000000..9860f60 --- /dev/null +++ b/docs/dep.txt @@ -0,0 +1,70 @@ +AccessControl --> /AccessControl/tests/testZopeGuards.py:471:class TestRestrictedPythonApply(GuardTestCase): +AccessControl --> /AccessControl/tests/testZopeGuards.py:517:# Given the high wall between AccessControl and RestrictedPython, I suppose +AccessControl --> /AccessControl/tests/testZopeGuards.py:601: from RestrictedPython.tests import verify +AccessControl --> /AccessControl/tests/testZopeGuards.py:631: from RestrictedPython.tests import verify +AccessControl --> /AccessControl/tests/testZopeGuards.py:658: from RestrictedPython.tests import verify +AccessControl --> /AccessControl/tests/testZopeGuards.py:680: from RestrictedPython.tests import verify +AccessControl --> /AccessControl/tests/testZopeGuards.py:702: from RestrictedPython.tests import verify +AccessControl --> /AccessControl/tests/testZopeGuards.py:722: from RestrictedPython.tests import verify +AccessControl --> /AccessControl/tests/testZopeGuards.py:748: from RestrictedPython import compile_restricted +AccessControl --> /AccessControl/tests/testZopeGuards.py:764: from RestrictedPython import compile_restricted +AccessControl --> /AccessControl/tests/testZopeGuards.py:872: TestRestrictedPythonApply, + +AccessControl --> /AccessControl/ZopeGuards.py:20:import RestrictedPython +AccessControl --> /AccessControl/ZopeGuards.py:21:from RestrictedPython.Guards import safe_builtins, full_write_guard +AccessControl --> /AccessControl/ZopeGuards.py:22:from RestrictedPython.Utilities import utility_builtins +AccessControl --> /AccessControl/ZopeGuards.py:23:from RestrictedPython.Eval import RestrictionCapableEval +AccessControl --> /AccessControl/ZopeGuards.py:540: '_print_': RestrictedPython.PrintCollector, + +collective.themefragments --> collective/themefragments/traversal.py:6:from RestrictedPython import compile_restricted_function + +DocumentTemplate --> DocumentTemplate/DT_Util.py:25:from RestrictedPython.Guards import safe_builtins +DocumentTemplate --> DocumentTemplate/DT_Util.py:26:from RestrictedPython.Utilities import utility_builtins +DocumentTemplate --> DocumentTemplate/DT_Util.py:27:from RestrictedPython.Eval import RestrictionCapableEval +DocumentTemplate --> DocumentTemplate/DT_Util.py:38: from RestrictedPython.Utilities import test +DocumentTemplate --> DocumentTemplate/DT_Util.py:79: from RestrictedPython.Limits import limited_builtins + +five.pt-2.2.3-py2.7.egg/five/pt/expressions.py:23:from RestrictedPython.RestrictionMutator import RestrictionMutator +five.pt-2.2.3-py2.7.egg/five/pt/expressions.py:24:from RestrictedPython.Utilities import utility_builtins +five.pt-2.2.3-py2.7.egg/five/pt/expressions.py:25:from RestrictedPython import MutatingWalker + +Products.CMFPlone-4.3.9-py2.7.egg/Products/CMFPlone/tests/testSecurity.py:32: def test_PT_allow_module_not_available_in_RestrictedPython_1(self): +Products.CMFPlone-4.3.9-py2.7.egg/Products/CMFPlone/tests/testSecurity.py:47: def test_PT_allow_module_not_available_in_RestrictedPython_2(self): +Products.CMFPlone-4.3.9-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:16:class RestrictedPythonTest(ZopeTestCase.ZopeTestCase): +Products.CMFPlone-4.3.9-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:38:class TestSecurityDeclarations(RestrictedPythonTest): +Products.CMFPlone-4.3.9-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:343:class TestAcquisitionMethods(RestrictedPythonTest): +Products.CMFPlone-4.3.9-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:432:class TestNavtreeSecurity(PloneTestCase.PloneTestCase, RestrictedPythonTest): + +Products.CMFPlone-5.0.2-py2.7.egg/Products/CMFPlone/tests/testSecurity.py:30: def test_PT_allow_module_not_available_in_RestrictedPython_1(self): +Products.CMFPlone-5.0.2-py2.7.egg/Products/CMFPlone/tests/testSecurity.py:45: def test_PT_allow_module_not_available_in_RestrictedPython_2(self): +Products.CMFPlone-5.0.2-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:15:class RestrictedPythonTest(TestCase): +Products.CMFPlone-5.0.2-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:49:class TestSecurityDeclarations(RestrictedPythonTest): +Products.CMFPlone-5.0.2-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:325:class TestAcquisitionMethods(RestrictedPythonTest): +Products.CMFPlone-5.0.2-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:400:class TestNavtreeSecurity(PloneTestCase.PloneTestCase, RestrictedPythonTest): + +Products.CMFPlone-5.1a1-py2.7.egg/Products/CMFPlone/tests/testSecurity.py:30: def test_PT_allow_module_not_available_in_RestrictedPython_1(self): +Products.CMFPlone-5.1a1-py2.7.egg/Products/CMFPlone/tests/testSecurity.py:45: def test_PT_allow_module_not_available_in_RestrictedPython_2(self): +Products.CMFPlone-5.1a1-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:15:class RestrictedPythonTest(TestCase): +Products.CMFPlone-5.1a1-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:49:class TestSecurityDeclarations(RestrictedPythonTest): +Products.CMFPlone-5.1a1-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:325:class TestAcquisitionMethods(RestrictedPythonTest): +Products.CMFPlone-5.1a1-py2.7.egg/Products/CMFPlone/tests/testSecurityDeclarations.py:400:class TestNavtreeSecurity(PloneTestCase.PloneTestCase, RestrictedPythonTest): + +Products.PythonScripts --> Products/PythonScripts/PythonScript.py:45:from RestrictedPython import compile_restricted_function +Products.PythonScripts --> Products/PythonScripts/tests/testPythonScript.py:18:from RestrictedPython.tests.verify import verify + +Products.ZCatalog --> /Products/PluginIndexes/TopicIndex/FilteredSet.py:19:from RestrictedPython.Eval import RestrictionCapableEval + +zope.security-3.7.4-py2.7-macosx-10.10-x86_64.egg/zope/security/untrustedpython/rcompile.py:23:import RestrictedPython.RCompile +zope.security-3.7.4-py2.7-macosx-10.10-x86_64.egg/zope/security/untrustedpython/rcompile.py:24:from RestrictedPython.SelectCompiler import ast, OP_ASSIGN, OP_DELETE, OP_APPLY +zope.security-3.7.4-py2.7-macosx-10.10-x86_64.egg/zope/security/untrustedpython/rcompile.py:33:class RExpression(RestrictedPython.RCompile.RestrictedCompileMode): +zope.security-3.7.4-py2.7-macosx-10.10-x86_64.egg/zope/security/untrustedpython/rcompile.py:39: RestrictedPython.RCompile.RestrictedCompileMode.__init__( +zope.security-3.7.4-py2.7-macosx-10.10-x86_64.egg/zope/security/untrustedpython/rcompile.txt:28:The implementation makes use of the `RestrictedPython` package, + +zope.untrustedpython --> rcompile.py:21:import RestrictedPython.RCompile +zope.untrustedpython --> rcompile.py:22:from RestrictedPython.SelectCompiler import ast +zope.untrustedpython --> rcompile.py:31:class RExpression(RestrictedPython.RCompile.RestrictedCompileMode): +zope.untrustedpython --> rcompile.py:37: RestrictedPython.RCompile.RestrictedCompileMode.__init__( + +* Products.PageTemplates --> ZRPythonExpr.py:15:Handler for Python expressions that uses the RestrictedPython package. +* Products.PageTemplates --> ZRPythonExpr.py:20:from RestrictedPython import compile_restricted_eval +* Products.PageTemplates --> ZRPythonExpr.py:32: # Unicode expression are not handled properly by RestrictedPython diff --git a/docs/idea.rst b/docs/idea.rst new file mode 100644 index 0000000..c100b84 --- /dev/null +++ b/docs/idea.rst @@ -0,0 +1,86 @@ +The Idea behind RestrictedPython +================================ + +Python is a `Turing complete`_ programming language. +To offer a Python interface for users in web context is a potential security risk. +Web frameworks and Content Management Systems (CMS) want to offer their users as much extensibility as possible through the web (TTW). +This also means to have permissions to add functionality via a Python Script. + +There should be additional preventive measures taken to ensure integrity of the application and the server itself, according to information security best practice and unrelated to Restricted Python. + +RestrictedPython defines a safe subset of the Python programming language. +This is a common approach for securing a programming language. +The `Ada Ravenscar profile`_ is another example of such an approach. + +Defining a secure subset of the language involves restricting the `EBNF`_ elements and explicitly allowing or disallowing language features. +Much of the power of a programming language derives from its standard and contributed libraries, so any calling of these methods must also be checked and potentially restricted. +RestrictedPython generally disallows calls to any library that is not explicit whitelisted. + +As Python is a scripting language that is executed by an interpreter. +Any Python code that should be executed have to be explicit checked before executing a generated byte code by the interpreter. + +Python itself offers three methods that provide such a workflow: + +* ``compile()`` which compiles source code to byte code +* ``exec`` / ``exec()`` which executes the byte code in the interpreter +* ``eval`` / ``eval()`` which executes a byte code expression + +Therefore RestrictedPython offers a replacement for the python builtin function ``compile()`` (Python 2: https://docs.python.org/2/library/functions.html#compile / Python 3 https://docs.python.org/3/library/functions.html#compile). +This method is defined as following: + +.. code:: Python + + compile(source, filename, mode [, flags [, dont_inherit]]) + +The definition of the ``compile()`` method has changed over time, but its relevant parameters ``source`` and ``mode`` still remain. + +There are three valid string values for ``mode``: + +* ``'exec'`` +* ``'eval'`` +* ``'single'`` + +For RestrictedPython this ``compile()`` method is replaced by: + +.. code:: Python + + compile_restricted(source, filename, mode [, flags [, dont_inherit]]) + +The primary parameter ``source`` has to be a ASCII or ``unicode`` string (With Python 2.6 an additional option for source was added: ``ast.AST`` for :ref:`Code generation <_sec_code_generation>`). +Both methods either returns compiled byte code that the interpreter could execute or raise exceptions if the provided source code is invalid. + +As ``compile`` and ``compile_restricted`` just compile the provided source code to byte code it is not sufficient to sandbox the environment, as all calls to libraries are still available. + +The two methods / Statements: + +* ``exec`` / ``exec()`` +* ``eval`` / ``eval()`` + +have two parameters: + +* ``globals`` +* ``locals`` + +which are a reference to the Python builtins. + +By modifying and restricting the available modules, methods and constants from globals and locals we could limit the possible calls. + +Additionally RestrictedPython offers a way to define a policy which allows developers to protect access to attributes. +This works by defining a restricted version of: + +* ``print`` +* ``getattr`` +* ``setattr`` +* ``import`` + +Also RestrictedPython provides three predefined, limited versions of Python's own ``__builtins__``: + +* ``safe_builtins`` (by Guards.py) +* ``limited_builtins`` (by Limits.py), which provides restricted sequence types +* ``utilities_builtins`` (by Utilities.py), which provides access for standard modules math, random, string and for sets. + +Additional there exist guard functions to make attributes of Python objects immutable --> ``full_write_guard`` (write and delete protected) + +.. _`Turing complete`: https://en.wikipedia.org/wiki/Turing_completeness +.. _Ada Ravenscar Profile: https://en.wikipedia.org/wiki/Ravenscar_profile +.. _EBNF: https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..2459200 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,38 @@ +.. RestrictedPython documentation master file, created by + sphinx-quickstart on Thu May 19 12:43:20 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +============================================ +Welcome to RestrictedPython's documentation! +============================================ + +.. include:: idea.rst + +Contents +======== + +.. toctree:: + :maxdepth: 2 + + idea + basics/index + usage/index + api/index + + RestrictedPython3/index + RestrictedPython4/index + upgrade/index + upgrade_dependencies/index + + roadmap/index + contributing/index + + CHANGES + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/install/index.rst b/docs/install/index.rst new file mode 100644 index 0000000..0286941 --- /dev/null +++ b/docs/install/index.rst @@ -0,0 +1,10 @@ +Install / Depend on RestrictedPython +==================================== + +RestrictedPython is usually not used stand alone, if you use it in context of your package add it to ``install_requires`` in your ``setup.py`` or a ``requirement.txt`` used by ``pip``. + +For a standalone usage: + +.. code:: bash + + pip install RestrictedPython diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..ab4e68c --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,281 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source +set I18NSPHINXOPTS=%SPHINXOPTS% source +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. epub3 to make an epub3 + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + echo. dummy to check syntax errors of document sources + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 1>NUL 2>NUL +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\RestrictedPython.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\RestrictedPython.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "epub3" ( + %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +if "%1" == "dummy" ( + %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. Dummy builder generates no files. + goto end +) + +:end diff --git a/src/RestrictedPython/notes.txt b/docs/notes.rst similarity index 79% rename from src/RestrictedPython/notes.txt rename to docs/notes.rst index a12da05..438122d 100644 --- a/src/RestrictedPython/notes.txt +++ b/docs/notes.rst @@ -1,6 +1,9 @@ How it works ============ +*Caution:* This is old documentation from RestrictedPython 3 and before. +Information should be transferred and this file should be removed. + Every time I see this code, I have to relearn it. These notes will hopefully make this a little easier. :) @@ -14,9 +17,9 @@ hopefully make this a little easier. :) and by DTML indirectly through Eval.RestrictionCapableEval. - OK, so lets see how this works by following the logic of - compile_restricted_eval. + compile_restricted_eval. - - First, we create an RExpression, passing the source and a + - First, we create an RExpression, passing the source and a "file name", to be used in tracebacks. Now, an RExpression is just: @@ -27,24 +30,24 @@ hopefully make this a little easier. :) mode to 'eval' and everided compile. Sigh. + RestrictedCompileMode is a subclass of AbstractCompileMode - that changes a bunch of things. :) These include compile, so we + that changes a bunch of things. :) These include compile, so we can ignore the compile we got from Expression. It would have been simpler to just set the dang mode in RExpression. Sigh. - RestrictedCompileMode seem to be the interestng base class. I + RestrictedCompileMode seem to be the interesting base class. I assume it implements the interesting functionality. We'll see below... - Next, we call compileAndTuplize. - + This calls compile on the RExpression. It has an error + + This calls compile on the RExpression. It has an error handler that does something that I hope I don't care about. :) - + It then calls the genCode method on the RExpression. This is - boring, so we'll not worry about it. + + It then calls the genCode method on the RExpression. + This is boring, so we'll not worry about it. - The compile method provided by RestrictedCompileMode is - interesting. + interesting. + First it calls _get_tree. @@ -55,10 +58,10 @@ hopefully make this a little easier. :) RestrictionMutator. The RestrictionMutator has the recipies for mutating the parse - tree. (Note, for comparison, that Zope3's - zope.security.untrustedpython.rcompile module an alternative + tree. (Note, for comparison, that Zope3's + zope.untrustedpython.rcompile module an alternative RestrictionMutator that provides a much smaller set of - changes.) + changes.) A mutator has visit method for different kinds of AST nodes. These visit methods may mutate nodes or return new @@ -70,9 +73,9 @@ hopefully make this a little easier. :) the given tree. Note _get_tree ignores the walk return value, thus assuming that the visitor for the root node doesn't return a new node. This is a theoretical bug that we can - ignore. + ignore. - + Second, it generates the code. This too is boring. + + Second, it generates the code. This too is boring. - So this seems simple enough. ;) When we want to add a check, we need to update or add a visit function in RestrictionMutator. @@ -80,7 +83,7 @@ hopefully make this a little easier. :) How does a visit function work. - First, we usually call walker.defaultVisitNode(node). This - transforms the node's child nodes. + transforms the node's child nodes. - Then we hack the node, or possibly return the node. To do this, we have to know how the node works. diff --git a/docs/roadmap/index.rst b/docs/roadmap/index.rst new file mode 100644 index 0000000..3fd98aa --- /dev/null +++ b/docs/roadmap/index.rst @@ -0,0 +1,39 @@ +Roadmap for RestrictedPython +============================ + +RestrictedPython 4.0 +-------------------- + +A feature complete rewrite of RestrictedPython using ``ast`` module instead of ``compile`` package. +RestrictedPython 4.0 should not add any new or remove restrictions. + +A detailed documentation that support usage and further development. + +Full code coverage tests. + +.. todo:: + + Complete documentation of all public API elements with docstyle comments + https://www.python.org/dev/peps/pep-0257/ + http://www.sphinx-doc.org/en/stable/ext/autodoc.html + http://thomas-cokelaer.info/tutorials/sphinx/docstring_python.html + +.. todo:: + + Resolve Discussion in https://github.com/zopefoundation/RestrictedPython/pull/39#issuecomment-283074699 + + compile_restricted optional params flags and dont_inherit will not work as expected with the current implementation. + + stephan-hof did propose a solution, should be discussed and if approved implemented. + + +RestrictedPython 4.1+ +--------------------- + +Enhance RestrictedPython, declare deprecations and possible new restrictions. + +RestrictedPython 5.0+ +--------------------- + +* Python 3+ only, no more support for Python 2.7 +* mypy - Static Code Analysis Annotations diff --git a/docs/upgrade/ast/python2_6.ast b/docs/upgrade/ast/python2_6.ast new file mode 100644 index 0000000..3ea12f8 --- /dev/null +++ b/docs/upgrade/ast/python2_6.ast @@ -0,0 +1,142 @@ +-- Python 2.6 AST +-- ASDL's five builtin types are identifier, int, string, object, bool + +module Python version "2.6" +{ + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list) + | ClassDef(identifier name, expr* bases, stmt* body, expr *decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- not sure if bool is allowed, can always use int + | Print(expr? dest, expr* values, bool nl) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + -- 'type' is a bad name + | Raise(expr? type, expr? inst, expr? tback) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier module, alias* names, int? level) + + -- Doesn't capture requirement that locals must be + -- defined if globals is + -- still supports use as a function! + | Exec(expr body, expr? globals, expr? locals) + + | Global(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | ListComp(expr elt, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Repr(expr value) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Ellipsis + | Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, expr? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (expr* args, identifier? vararg, + identifier? kwarg, expr* defaults) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) +} diff --git a/docs/upgrade/ast/python2_7.ast b/docs/upgrade/ast/python2_7.ast new file mode 100644 index 0000000..899cdc5 --- /dev/null +++ b/docs/upgrade/ast/python2_7.ast @@ -0,0 +1,144 @@ +-- Python 2.7 AST +-- ASDL's five builtin types are identifier, int, string, object, bool + +module Python version "2.7" +{ + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list) + | ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- not sure if bool is allowed, can always use int + | Print(expr? dest, expr* values, bool nl) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + -- 'type' is a bad name + | Raise(expr? type, expr? inst, expr? tback) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + -- Doesn't capture requirement that locals must be + -- defined if globals is + -- still supports use as a function! + | Exec(expr body, expr? globals, expr? locals) + + | Global(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Repr(expr value) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Ellipsis + | Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, expr? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (expr* args, identifier? vararg, + identifier? kwarg, expr* defaults) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) +} diff --git a/docs/upgrade/ast/python3_0.ast b/docs/upgrade/ast/python3_0.ast new file mode 100644 index 0000000..44d6e18 --- /dev/null +++ b/docs/upgrade/ast/python3_0.ast @@ -0,0 +1,147 @@ +-- Python 3.0 AST +-- ASDL's four builtin types are identifier, int, string, object + +module Python version "3.0" +{ + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr *decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + | Raise(expr? exc, expr? cause) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(string s) + | Ellipsis + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, identifier? vararg, expr? varargannotation, + arg* kwonlyargs, identifier? kwarg, + expr? kwargannotation, expr* defaults, + expr* kw_defaults) + arg = (identifier arg, expr? annotation) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) +} diff --git a/docs/upgrade/ast/python3_1.ast b/docs/upgrade/ast/python3_1.ast new file mode 100644 index 0000000..a0bbfb6 --- /dev/null +++ b/docs/upgrade/ast/python3_1.ast @@ -0,0 +1,149 @@ +-- Python 3.1 AST +-- ASDL's four builtin types are identifier, int, string, object + +module Python version "3.1" +{ + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr *decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + | Raise(expr? exc, expr? cause) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(string s) + | Ellipsis + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, identifier? vararg, expr? varargannotation, + arg* kwonlyargs, identifier? kwarg, + expr? kwargannotation, expr* defaults, + expr* kw_defaults) + arg = (identifier arg, expr? annotation) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) +} diff --git a/docs/upgrade/ast/python3_2.ast b/docs/upgrade/ast/python3_2.ast new file mode 100644 index 0000000..054d63e --- /dev/null +++ b/docs/upgrade/ast/python3_2.ast @@ -0,0 +1,146 @@ +-- Python 3.2 AST +-- ASDL's four builtin types are identifier, int, string, object + +module Python version "3.2" +{ + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(expr context_expr, expr? optional_vars, stmt* body) + + | Raise(expr? exc, expr? cause) + | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) + | TryFinally(stmt* body, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(string s) + | Ellipsis + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load | Store | Del | AugLoad | AugStore | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + -- not sure what to call the first argument for raise and except + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, identifier? vararg, expr? varargannotation, + arg* kwonlyargs, identifier? kwarg, + expr? kwargannotation, expr* defaults, + expr* kw_defaults) + arg = (identifier arg, expr? annotation) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) +} diff --git a/docs/upgrade/ast/python3_3.ast b/docs/upgrade/ast/python3_3.ast new file mode 100644 index 0000000..ad9e258 --- /dev/null +++ b/docs/upgrade/ast/python3_3.ast @@ -0,0 +1,155 @@ +-- PYTHON 3.3 AST +-- ASDL's five builtin types are identifier, int, string, bytes, object + +module Python version "3.3" +{ + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + | YieldFrom(expr value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(bytes s) + | Ellipsis + -- other literals? bools? + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, + identifier? vararg, + expr? varargannotation, + arg* kwonlyargs, + identifier? kwarg, + expr? kwargannotation, + expr* defaults, + expr* kw_defaults) + arg = (identifier arg, expr? annotation) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) + + withitem = (expr context_expr, expr? optional_vars) +} diff --git a/docs/upgrade/ast/python3_4.ast b/docs/upgrade/ast/python3_4.ast new file mode 100644 index 0000000..edfb756 --- /dev/null +++ b/docs/upgrade/ast/python3_4.ast @@ -0,0 +1,156 @@ +-- Python 3.4 AST +-- ASDL's six builtin types are identifier, int, string, bytes, object, singleton + +module Python version "3.4" +{ + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns) + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + expr? starargs, + expr? kwargs, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Yield(expr? value) + | YieldFrom(expr value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords, + expr? starargs, expr? kwargs) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(bytes s) + | NameConstant(singleton value) + | Ellipsis + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + excepthandler = ExceptHandler(expr? type, + identifier? name, + stmt* body) + attributes (int lineno, + int col_offset) + + arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, + arg? kwarg, expr* defaults) + + arg = (identifier arg, expr? annotation) + attributes (int lineno, int col_offset) + + -- keyword arguments supplied to call + keyword = (identifier arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) + + withitem = (expr context_expr, expr? optional_vars) +} diff --git a/docs/upgrade/ast/python3_5.ast b/docs/upgrade/ast/python3_5.ast new file mode 100644 index 0000000..dfe5bf1 --- /dev/null +++ b/docs/upgrade/ast/python3_5.ast @@ -0,0 +1,152 @@ +-- Python 3.5 AST +-- ASDL's six builtin types are identifier, int, string, bytes, object, singleton + +module Python version "3.5" +{ + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) + | AsyncFunctionDef(identifier name, arguments args, + stmt* body, expr* decorator_list, expr? returns) + + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + | AsyncWith(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass | Break | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Await(expr value) + | Yield(expr? value) + | YieldFrom(expr value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | Bytes(bytes s) + | NameConstant(singleton value) + | Ellipsis + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | MatMult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs) + + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, + arg? kwarg, expr* defaults) + + arg = (identifier arg, expr? annotation) + attributes (int lineno, int col_offset) + + -- keyword arguments supplied to call (NULL identifier for **kwargs) + keyword = (identifier? arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) + + withitem = (expr context_expr, expr? optional_vars) +} diff --git a/docs/upgrade/ast/python3_6.ast b/docs/upgrade/ast/python3_6.ast new file mode 100644 index 0000000..6cf106a --- /dev/null +++ b/docs/upgrade/ast/python3_6.ast @@ -0,0 +1,169 @@ +-- Python 3.6 AST +-- ASDL's 7 builtin types are: +-- identifier, int, string, bytes, object, singleton, constant +-- +-- singleton: None, True or False +-- constant can be None, whereas None means "no value" for object. + +module Python version "3.6" +{ + mod = Module(stmt* body) + | Interactive(stmt* body) + | Expression(expr body) + + -- not really an actual node but useful in Jython's typesystem. + | Suite(stmt* body) + + stmt = FunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns) + | AsyncFunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns) + + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + stmt* body, + expr* decorator_list) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value) + | AugAssign(expr target, operator op, expr value) + -- 'simple' indicates that we annotate simple name without parens + | AnnAssign(expr target, expr annotation, expr? value, int simple) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body) + | AsyncWith(withitem* items, stmt* body) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- XXX Jython will be different + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Await(expr value) + | Yield(expr? value) + | YieldFrom(expr value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords) + | Num(object n) -- a number as a PyObject. + | Str(string s) -- need to specify raw, unicode, etc? + | FormattedValue(expr value, int? conversion, expr? format_spec) + | JoinedStr(expr* values) + | Bytes(bytes s) + | NameConstant(singleton value) + | Ellipsis + | Constant(constant value) + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, slice slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) + + expr_context = Load + | Store + | Del + | AugLoad + | AugStore + | Param + + slice = Slice(expr? lower, expr? upper, expr? step) + | ExtSlice(slice* dims) + | Index(expr value) + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | MatMult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs, int is_async) + + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset) + + arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, + arg? kwarg, expr* defaults) + + arg = (identifier arg, expr? annotation) + attributes (int lineno, int col_offset) + + -- keyword arguments supplied to call (NULL identifier for **kwargs) + keyword = (identifier? arg, expr value) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) + + withitem = (expr context_expr, expr? optional_vars) +} diff --git a/docs/upgrade/index.rst b/docs/upgrade/index.rst new file mode 100644 index 0000000..9ce393e --- /dev/null +++ b/docs/upgrade/index.rst @@ -0,0 +1,115 @@ +Concept for a upgrade to Python 3 +================================= + +RestrictedPython is a classic approach of compiler construction to create a limited subset of an existing programming language. + +Defining a programming language requires a regular grammar (`Chomsky 3`_ / `EBNF`_) definition. +This grammar will be implemented in an abstract syntax tree (AST), which will be passed on to a code generator to produce a machine-readable version. + +.. _`_sec_code_generation`: + +Code generation +--------------- + +As Python is a platform independent programming language, this machine readable version is a byte code which will be translated on the fly by an interpreter into machine code. +This machine code then gets executed on the specific CPU architecture, with the standard operating system restrictions. + +The byte code produced must be compatible with the execution environment that the Python interpreter is running in, so we do not generate the byte code directly from ``compile_restricted`` and the other ``compile_restricted_*`` methods manually, it may not match what the interpreter expects. + +Thankfully, the Python ``compile()`` function introduced the capability to compile ``ast.AST`` code into byte code in Python 2.6, so we can return the platform-independent AST and keep byte code generation delegated to the interpreter. + +``compiler.ast`` --> ``ast`` +---------------------------- + +As the ``compiler`` module was deprecated in Python 2.6 and was removed before Python 3.0 was released it has never been available for any Python 3 version. +Instead Python 2.6 / Python 3 introduced the new ``ast`` module, that is more widly supported. +So we need to move from ``compiler.ast`` to ``ast`` to support newer Python versions. + +From the point of view of compiler design, the concepts of the ``compiler`` module and the ``ast`` module are similar. +The ``compiler`` module predates several major improvements of the Python development like a generally applied style guide. +While ``compiler`` still uses the old `CamelCase`_ Syntax (``visitNode(self, node, walker)``) the ``ast.AST`` did now use the Python common ``visit_Node(self, node)`` syntax. +Also the names of classes have been changed, where ``compiler`` uses ``Walker`` and ``Mutator`` the corresponding elements in ``ast.AST`` are ``NodeVisitor`` and ``NodeTransformator``. + + +``ast`` module (Abstract Syntax Trees) +-------------------------------------- + +The ``ast`` module consists of four areas: + +* ``AST`` (Basis of all Nodes) + all node class implementations +* ``NodeVisitor`` and ``NodeTransformer`` (tool to consume and modify the AST) +* Helper methods + + * ``parse`` + * ``walk`` + * ``dump`` + +* Constants + + * ``PyCF_ONLY_AST`` + + +``NodeVisitor`` & ``NodeTransformer`` +..................................... + +A ``NodeVisitor`` is a class of a node / AST consumer, it reads the data by stepping through the tree without modifying it. +In contrast, a ``NodeTransformer`` (which inherits from a ``NodeVisitor``) is allowed to modify the tree and nodes. + +Modifying the AST +----------------- + + + + + + + + +Technical Backgrounds - Links to External Documentation +--------------------------------------------------------- + +* Concept of Immutable Types and Python Example (https://en.wikipedia.org/wiki/Immutable_object#Python) +* Python 3 Standard Library Documentation on AST module ``ast`` (https://docs.python.org/3/library/ast.html) + + * AST Grammar of Python + + * `Python 3.6 AST`_ + * `Python 3.5 AST`_ + * `Python 3.4 AST`_ + * `Python 3.3 AST`_ + * `Python 3.2 AST`_ + * `Python 3.1 AST`_ + * `Python 3.0 AST`_ + * `Python 2.7 AST`_ + * `Python 2.6 AST`_ + + * ``NodeVistiors`` (https://docs.python.org/3.5/library/ast.html#ast.NodeVisitor) + * ``NodeTransformer`` (https://docs.python.org/3.5/library/ast.html#ast.NodeTransformer) + * ``dump`` (https://docs.python.org/3.5/library/ast.html#ast.dump) + +* In detail Documentation on the Python AST module ``ast`` (Green Tree Snakes) (https://greentreesnakes.readthedocs.org/en/latest/) +* Example how to Instrumenting the Python AST (``ast.AST``) (http://www.dalkescientific.com/writings/diary/archive/2010/02/22/instrumenting_the_ast.html) + +.. _`CamelCase`: https://en.wikipedia.org/wiki/Camel_case + +.. _`EBNF`: https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form + +.. _`Chomsky 3`: https://en.wikipedia.org/wiki/Chomsky_hierarchy#Type-3_grammars + +.. _`Python 3.6 AST`: https://docs.python.org/3.6/library/ast.html#abstract-grammar + +.. _`Python 3.5 AST`: https://docs.python.org/3.5/library/ast.html#abstract-grammar + +.. _`Python 3.4 AST`: https://docs.python.org/3.4/library/ast.html#abstract-grammar + +.. _`Python 3.3 AST`: https://docs.python.org/3.3/library/ast.html#abstract-grammar + +.. _`Python 3.2 AST`: https://docs.python.org/3.2/library/ast.html#abstract-grammar + +.. _`Python 3.1 AST`: https://docs.python.org/3.1/library/ast.html#abstract-grammar + +.. _`Python 3.0 AST`: https://docs.python.org/3.0/library/ast.html#abstract-grammar + +.. _`Python 2.7 AST`: https://docs.python.org/2.7/library/ast.html#abstract-grammar + +.. _`Python 2.6 AST`: https://docs.python.org/2.6/library/ast.html#abstract-grammar diff --git a/docs/upgrade_dependencies/index.rst b/docs/upgrade_dependencies/index.rst new file mode 100644 index 0000000..d411a14 --- /dev/null +++ b/docs/upgrade_dependencies/index.rst @@ -0,0 +1,42 @@ +Upgrade dependencies +==================== + +The following packages used in Zope2 and Plone depend on ``RestricedPython``: + +* AccessControl +* zope.untrustedpython +* DocumentTemplate +* Products.PageTemplates +* Products.PythonScripts +* Products.PluginIndexes +* five.pt (wrapping some functions and protection for Chameleon) + +Upgrade path +------------ + +For packages that use RestrictedPython the upgrade path differs on the actual usage. +If it uses pure RestrictedPython without any additional checks it should be just to check the imports. +RestrictedPython did move some of the imports to the base namespace, so you should only import directly from ``RestrictedPython.__init__.py``. + +* compile_restricted methods: + + * ``from RestrictedPython import compile_restricted`` + * ``from RestrictedPython import compile_restricted_eval`` + * ``from RestrictedPython import compile_restricted_exec`` + * ``from RestrictedPython import compile_restricted_function`` + * ``from RestrictedPython import compile_restricted_single`` + +* predefined built-ins: + + * ``from RestrictedPython import safe_builtins`` + * ``from RestrictedPython import limited_builtins`` + * ``from RestrictedPython import utility_builtins`` + +* helper methods: + + * ``from RestrictedPython import PrintCollector`` + +Any import from ``RestrictedPython.RCompile`` indicates that there have been advanced checks implemented. +Those advanced checks where implemented via a ``MutatingWalker``. +Any checks needs to be reimplemented as a subclass of +``RestrictingNodeTransformer``. diff --git a/docs/usage/api.rst b/docs/usage/api.rst new file mode 100644 index 0000000..086efab --- /dev/null +++ b/docs/usage/api.rst @@ -0,0 +1,108 @@ +API overview +------------ + +RestrictedPython has tree major scopes: + +1. ``compile_restricted`` methods: + +.. py:method:: compile_restricted(source, filename, mode, flags, dont_inherit, policy) + :module: RestrictedPython + + Compiles source code into interpretable byte code. + + :param source: (required). The source code that should be compiled + :param filename: (optional). + :param mode: (optional). + :param flags: (optional). + :param dont_inherit: (optional). + :param policy: (optional). + :type source: str or unicode text + :type filename: str or unicode text + :type mode: str or unicode text + :type flags: int + :type dont_inherit: int + :type policy: RestrictingNodeTransformer class + :return: Byte Code + +.. py:method:: compile_restricted_exec(source, filename, flags, dont_inherit, policy) + :module: RestrictedPython + + Compiles source code into interpretable byte code. + + :param source: (required). The source code that should be compiled + :param filename: (optional). + :param flags: (optional). + :param dont_inherit: (optional). + :param policy: (optional). + :type source: str or unicode text + :type filename: str or unicode text + :type mode: str or unicode text + :type flags: int + :type dont_inherit: int + :type policy: RestrictingNodeTransformer class + :return: CompileResult (a namedtuple with code, errors, warnings, used_names) + +.. py:method:: compile_restricted_eval(source, filename, flags, dont_inherit, policy) + :module: RestrictedPython + + Compiles source code into interpretable byte code. + + :param source: (required). The source code that should be compiled + :param filename: (optional). + :param flags: (optional). + :param dont_inherit: (optional). + :param policy: (optional). + :type source: str or unicode text + :type filename: str or unicode text + :type mode: str or unicode text + :type flags: int + :type dont_inherit: int + :type policy: RestrictingNodeTransformer class + :return: CompileResult (a namedtuple with code, errors, warnings, used_names) + +.. py:method:: compile_restricted_single(source, filename, flags, dont_inherit, policy) + :module: RestrictedPython + + Compiles source code into interpretable byte code. + + :param source: (required). The source code that should be compiled + :param filename: (optional). + :param flags: (optional). + :param dont_inherit: (optional). + :param policy: (optional). + :type source: str or unicode text + :type filename: str or unicode text + :type mode: str or unicode text + :type flags: int + :type dont_inherit: int + :type policy: RestrictingNodeTransformer class + :return: CompileResult (a namedtuple with code, errors, warnings, used_names) + +.. py:method:: compile_restricted_function(p, body, name, filename, globalize=None) + :module: RestrictedPython + + Compiles source code into interpretable byte code. + + :param p: (required). + :param body: (required). + :param name: (required). + :param filename: (required). + :param globalize: (optional). + :type p: + :type body: + :type name: str or unicode text + :type filename: str or unicode text + :type globalize: + :return: byte code + + + +2. restricted builtins + + * ``safe_builtins`` + * ``limited_builtins`` + * ``utility_builtins`` + +3. helper modules + + * ``PrintCollector`` diff --git a/docs/usage/basic_usage.rst b/docs/usage/basic_usage.rst new file mode 100644 index 0000000..c7beff7 --- /dev/null +++ b/docs/usage/basic_usage.rst @@ -0,0 +1,104 @@ +Basic usage +----------- + +The general workflow to execute Python code that is loaded within a Python program is: + +.. testcode:: + + source_code = """ + def do_something(): + pass + """ + + byte_code = compile(source_code, filename='', mode='exec') + exec(byte_code) + do_something() + +With RestrictedPython that workflow should be as straight forward as possible: + +.. testcode:: + + from RestrictedPython import compile_restricted + + source_code = """ + def do_something(): + pass + """ + + byte_code = compile_restricted(source_code, + filename='', + mode='exec') + exec(byte_code) + do_something() + +You might also use the replacement import: + +.. testcode:: + + from RestrictedPython import compile_restricted as compile + +``compile_restricted`` uses a predefined policy that checks and modify the source code and checks against a restricted subset of the Python language. +The compiled source code is still executed against the full available set of library modules and methods. + +The Python :py:func:`exec` takes three parameters: + +* ``code`` which is the compiled byte code +* ``globals`` which is global dictionary +* ``locals`` which is the local dictionary + +By limiting the entries in the ``globals`` and ``locals`` dictionaries you +restrict the access to the available library modules and methods. + +Providing defined dictionaries for ``exec()`` should be used in context of RestrictedPython. + +.. code:: Python + + byte_code = + exec(byte_code, { ... }, { ... }) + +Typically there is a defined set of allowed modules, methods and constants used in that context. +RestrictedPython provides three predefined built-ins for that (see :ref:`predefined_builtins` for details): + +* ``safe_builtins`` +* ``limited_builtins`` +* ``utility_builtins`` + +So you normally end up using: + +.. testcode:: + + from RestrictedPython import compile_restricted + + from RestrictedPython import safe_builtins + from RestrictedPython import limited_builtins + from RestrictedPython import utility_builtins + + source_code = """ + def do_something(): + pass + """ + + try: + byte_code = compile_restricted(source_code, + filename='', + mode='exec') + + exec(byte_code, safe_builtins, None) + except SyntaxError as e: + pass + +One common advanced usage would be to define an own restricted builtin dictionary. + +Necessary setup +--------------- + +`RestrictedPython` requires some predefined names in globals in order to work +properly. + +To use classes in Python 3 + ``__metaclass__`` must be set. Set it to ``type`` to use no custom metaclass. + +To use ``for`` statements and comprehensions + ``_iter_unpack_sequence_`` must point to :func:`RestrictedPython.Guards.guarded_iter_unpack_sequence`. + +The usage of `RestrictedPython` in :mod:`AccessControl.ZopeGuards` can serve as example. diff --git a/docs/usage/framework_usage.rst b/docs/usage/framework_usage.rst new file mode 100644 index 0000000..0711d8a --- /dev/null +++ b/docs/usage/framework_usage.rst @@ -0,0 +1,85 @@ +.. _sec_usage_frameworks: + +Usage in frameworks and Zope +---------------------------- + +One major issue with using ``compile_restricted`` directly in a framework is, that you have to use try-except statements to handle problems and it might be a bit harder to provide useful information to the user. +RestrictedPython provides four specialized compile_restricted methods: + +* ``compile_restricted_exec`` +* ``compile_restricted_eval`` +* ``compile_restricted_single`` +* ``compile_restricted_function`` + +Those four methods return a named tuple (``CompileResult``) with four elements: + +* ``code`` ```` object or ``None`` if ``errors`` is not empty +* ``errors`` a tuple with error messages +* ``warnings`` a list with warnings +* ``used_names`` a set / dictionary with collected used names of library calls + +Those three information "lists" could be used to provide the user with informations about the compiled source code. + +Typical uses cases for the four specialized methods: + +* ``compile_restricted_exec`` --> Python Modules or Scripts that should be used or called by the framework itself or from user calls +* ``compile_restricted_eval`` --> Templates +* ``compile_restricted_single`` +* ``compile_restricted_function`` + +Modifying the builtins is straight forward, it is just a dictionary containing access pointers to available library elements. +Modification is normally removing elements from existing builtins or adding allowed elements by copying from globals. + +For frameworks it could possibly also be useful to change handling of specific Python language elements. +For that use case RestrictedPython provides the possibility to pass an own policy. + +A policy is basically a special ``NodeTransformer`` that could be instantiated with three params for ``errors``, ``warnings`` and ``used_names``, it should be a subclass of RestrictedPython.RestrictingNodeTransformer. + +.. testcode:: own_policy + + from RestrictedPython import compile_restricted + from RestrictedPython import RestrictingNodeTransformer + + class OwnRestrictingNodeTransformer(RestrictingNodeTransformer): + pass + + policy_instance = OwnRestrictingNodeTransformer(errors=[], + warnings=[], + used_names=[]) + +All ``compile_restricted*`` methods do have a optional parameter ``policy``, where a specific policy could be provided. + +.. testcode:: own_policy + + source_code = """ + def do_something(): + pass + """ + + policy = OwnRestrictingNodeTransformer + + byte_code = compile_restricted(source_code, + filename='', + mode='exec', + policy=policy # Policy Class + ) + exec(byte_code, globals(), None) + +One special case "unrestricted RestrictedPython" (defined to unblock ports of Zope Packages to Python 3) is to actually use RestrictedPython in an unrestricted mode, by providing a Null-Policy (aka ``None``). +That special case would be written as: + +.. testcode:: + + from RestrictedPython import compile_restricted + + source_code = """ + def do_something(): + pass + """ + + byte_code = compile_restricted(source_code, + filename='', + mode='exec', + policy=None # Null-Policy -> unrestricted + ) + exec(byte_code, globals(), None) diff --git a/docs/usage/index.rst b/docs/usage/index.rst new file mode 100644 index 0000000..88b25c6 --- /dev/null +++ b/docs/usage/index.rst @@ -0,0 +1,7 @@ +Usage of RestrictedPython +========================= + +.. include:: basic_usage.rst +.. include:: framework_usage.rst +.. include:: policy.rst +.. include:: api.rst diff --git a/docs/usage/policy.rst b/docs/usage/policy.rst new file mode 100644 index 0000000..245ebc7 --- /dev/null +++ b/docs/usage/policy.rst @@ -0,0 +1,46 @@ +.. _policy_builtins: + +Policies & builtins +------------------- + +.. todo:: + + Should be described in detail. + Especially the difference between builtins and a policy which is a NodeTransformer. + + +RestrictedPython provides a way to define Policies, by redefining restricted versions of ``print``, ``getattr``, ``setattr``, ``import``, etc.. +As shortcuts it offers three stripped down versions of Pythons ``__builtins__``: + +.. _predefined_builtins: + +Predefined builtins +................... + +.. todo:: + + Describe more in details + +* ``safe_builtins`` a safe set of builtin modules and functions, +* ``limited_builtins`` which provides restricted sequence types, +* ``utility_builtins`` which provides access for standard modules math, random, string and for sets. + +Guards +...... + +.. todo:: + + Describe Guards and predefined guard methods in details + +RestrictedPython predefines several guarded access and manipulation methods: + +* ``guarded_setattr`` +* ``guarded_delattr`` +* ``guarded_iter_unpack_sequence`` +* ``guarded_unpack_sequence`` + +Those and additional methods rely on a helper construct ``full_write_guard``, which is intended to help implement immutable and semi mutable objects and attributes. + +.. todo:: + + Describe full_write_guard more in detail and how it works. diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..9409106 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,54 @@ +[build_sphinx] +source-dir = docs/source +build-dir = docs +all_files = 1 + +[upload_sphinx] +upload-dir = docs/html + +[check-manifest] +ignore = + .travis.yml + bootstrap-buildout.py + bootstrap.py + buildout.cfg + jenkins.cfg + travis.cfg + +[aliases] +test = pytest + +[tool:pytest] +addopts = +testpaths = + tests + src/RestrictedPython/tests +norecursedirs = fixures + +isort_ignore = + bootstrap.py + + +[isort] +force_alphabetical_sort = True +force_single_line = True +lines_after_imports = 2 +line_length = 200 +skip = + bootstrap.py +not_skip = + __init__.py + +[flake8] +exclude = + bootstrap.py, + src/RestrictedPython/tests, + src/RestrictedPython/SelectCompiler.py, + +ignore = + N801, + N802, + N803, + N805, + N806, + N812, diff --git a/setup.py b/setup.py index 5e7362f..78d0bc2 100644 --- a/setup.py +++ b/setup.py @@ -11,29 +11,64 @@ # FOR A PARTICULAR PURPOSE. # ############################################################################## -"""Setup for RestrictedPython package -""" +"""Setup for RestrictedPython package""" + +from setuptools import find_packages +from setuptools import setup + import os -from setuptools import setup, find_packages + def read(*rnames): return open(os.path.join(os.path.dirname(__file__), *rnames)).read() + setup(name='RestrictedPython', - version='3.7.dev0', + version='4.0.0.dev0', url='http://pypi.python.org/pypi/RestrictedPython', license='ZPL 2.1', description='RestrictedPython provides a restricted execution ' - 'environment for Python, e.g. for running untrusted code.', + 'environment for Python, e.g. for running untrusted code.', + long_description=(read('README.rst') + '\n' + + read('docs', 'CHANGES.rst')), + classifiers=[ + 'License :: OSI Approved :: Zope Public License', + 'Programming Language :: Python', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Topic :: Security', + ], author='Zope Foundation and Contributors', author_email='zope-dev@zope.org', - long_description=(read('src', 'RestrictedPython', 'README.txt') - + '\n' + - read('CHANGES.txt')), + packages=find_packages('src'), + package_dir={'': 'src'}, + install_requires=[ + 'setuptools', - packages = find_packages('src'), - package_dir = {'': 'src'}, - install_requires = ['setuptools'], - include_package_data = True, - zip_safe = False, + ], + setup_requires=[ + 'pytest-runner', + ], + test_requires=[ + 'pytest', + ], + extras_require={ + 'docs': [ + 'Sphinx', + ], + 'release': [ + 'zest.releaser', + ], + 'develop': [ + 'pdbpp', + 'isort', + ], + }, + include_package_data=True, + zip_safe=False, ) diff --git a/src/RestrictedPython/Eval.py b/src/RestrictedPython/Eval.py index 9875f95..836ea5e 100644 --- a/src/RestrictedPython/Eval.py +++ b/src/RestrictedPython/Eval.py @@ -10,32 +10,42 @@ # FOR A PARTICULAR PURPOSE # ############################################################################## -"""Restricted Python Expressions -""" +"""Restricted Python Expressions.""" -__version__='$Revision: 1.6 $'[11:-2] +from ._compat import IS_PY2 +from .compile import compile_restricted_eval -from RestrictedPython import compile_restricted_eval +import ast -from string import translate, strip -import string -nltosp = string.maketrans('\r\n',' ') +if IS_PY2: + from string import maketrans +else: + maketrans = str.maketrans + + +nltosp = maketrans('\r\n', ' ') + +# No restrictions. +default_guarded_getattr = getattr -default_guarded_getattr = getattr # No restrictions. def default_guarded_getitem(ob, index): # No restrictions. return ob[index] -PROFILE = 0 -class RestrictionCapableEval: +class RestrictionCapableEval(object): """A base class for restricted code.""" - + globals = {'__builtins__': None} - rcode = None # restricted - ucode = None # unrestricted + # restricted + rcode = None + + # unrestricted + ucode = None + + # Names used by the expression used = None def __init__(self, expr): @@ -45,72 +55,60 @@ def __init__(self, expr): expr -- a string containing the expression to be evaluated. """ - expr = strip(expr) + expr = expr.strip() self.__name__ = expr - expr = translate(expr, nltosp) + expr = expr.translate(nltosp) self.expr = expr - self.prepUnrestrictedCode() # Catch syntax errors. + # Catch syntax errors. + self.prepUnrestrictedCode() def prepRestrictedCode(self): if self.rcode is None: - if PROFILE: - from time import clock - start = clock() - co, err, warn, used = compile_restricted_eval( - self.expr, '') - if PROFILE: - end = clock() - print 'prepRestrictedCode: %d ms for %s' % ( - (end - start) * 1000, `self.expr`) - if err: - raise SyntaxError, err[0] - self.used = tuple(used.keys()) - self.rcode = co + result = compile_restricted_eval(self.expr, '') + if result.errors: + raise SyntaxError(result.errors[0]) + self.used = tuple(result.used_names) + self.rcode = result.code def prepUnrestrictedCode(self): if self.ucode is None: - # Use the standard compiler. - co = compile(self.expr, '', 'eval') + exp_node = compile( + self.expr, + '', + 'eval', + ast.PyCF_ONLY_AST) + + co = compile(exp_node, '', 'eval') + + # Examine the ast to discover which names the expression needs. if self.used is None: - # Examine the code object, discovering which names - # the expression needs. - names=list(co.co_names) - used={} - i=0 - code=co.co_code - l=len(code) - LOAD_NAME=101 - HAVE_ARGUMENT=90 - while(i < l): - c=ord(code[i]) - if c==LOAD_NAME: - name=names[ord(code[i+1])+256*ord(code[i+2])] - used[name]=1 - i=i+3 - elif c >= HAVE_ARGUMENT: i=i+3 - else: i=i+1 - self.used=tuple(used.keys()) - self.ucode=co + used = set() + for node in ast.walk(exp_node): + if isinstance(node, ast.Name): + if isinstance(node.ctx, ast.Load): + used.add(node.id) + + self.used = tuple(used) + + self.ucode = co def eval(self, mapping): # This default implementation is probably not very useful. :-( # This is meant to be overridden. self.prepRestrictedCode() - code = self.rcode - d = {'_getattr_': default_guarded_getattr, - '_getitem_': default_guarded_getitem} - d.update(self.globals) - has_key = d.has_key + + global_scope = { + '_getattr_': default_guarded_getattr, + '_getitem_': default_guarded_getitem + } + + global_scope.update(self.globals) + for name in self.used: - try: - if not has_key(name): - d[name] = mapping[name] - except KeyError: - # Swallow KeyErrors since the expression - # might not actually need the name. If it - # does need the name, a NameError will occur. - pass - return eval(code, d) + if (name not in global_scope) and (name in mapping): + global_scope[name] = mapping[name] + + return eval(self.rcode, global_scope) def __call__(self, **kw): return self.eval(kw) diff --git a/src/RestrictedPython/Guards.py b/src/RestrictedPython/Guards.py index dfa1d65..21c8b4e 100644 --- a/src/RestrictedPython/Guards.py +++ b/src/RestrictedPython/Guards.py @@ -10,23 +10,126 @@ # FOR A PARTICULAR PURPOSE # ############################################################################## -__version__ = '$Revision: 1.14 $'[11:-2] - -import exceptions # This tiny set of safe builtins is extended by users of the module. # AccessControl.ZopeGuards contains a large set of wrappers for builtins. # DocumentTemplate.DT_UTil contains a few. +from ._compat import IS_PY2 + + +if IS_PY2: + import __builtin__ as builtins +else: + # Do not attempt to use this package on Python2.7 as there + # might be backports for this package such as future. + import builtins + safe_builtins = {} -for name in ['False', 'None', 'True', 'abs', 'basestring', 'bool', 'callable', - 'chr', 'cmp', 'complex', 'divmod', 'float', 'hash', - 'hex', 'id', 'int', 'isinstance', 'issubclass', 'len', - 'long', 'oct', 'ord', 'pow', 'range', 'repr', 'round', - 'str', 'tuple', 'unichr', 'unicode', 'xrange', 'zip']: +_safe_names = [ + 'None', + 'False', + 'True', + 'abs', + 'bool', + 'callable', + 'chr', + 'complex', + 'divmod', + 'float', + 'hash', + 'hex', + 'id', + 'int', + 'isinstance', + 'issubclass', + 'len', + 'oct', + 'ord', + 'pow', + 'range', + 'repr', + 'round', + 'slice', + 'str', + 'tuple', + 'zip' +] + +_safe_exceptions = [ + 'ArithmeticError', + 'AssertionError', + 'AttributeError', + 'BaseException', + 'BufferError', + 'BytesWarning', + 'DeprecationWarning', + 'EOFError', + 'EnvironmentError', + 'Exception', + 'FloatingPointError', + 'FutureWarning', + 'GeneratorExit', + 'IOError', + 'ImportError', + 'ImportWarning', + 'IndentationError', + 'IndexError', + 'KeyError', + 'KeyboardInterrupt', + 'LookupError', + 'MemoryError', + 'NameError', + 'NotImplementedError', + 'OSError', + 'OverflowError', + 'PendingDeprecationWarning', + 'ReferenceError', + 'RuntimeError', + 'RuntimeWarning', + 'StopIteration', + 'SyntaxError', + 'SyntaxWarning', + 'SystemError', + 'SystemExit', + 'TabError', + 'TypeError', + 'UnboundLocalError', + 'UnicodeDecodeError', + 'UnicodeEncodeError', + 'UnicodeError', + 'UnicodeTranslateError', + 'UnicodeWarning', + 'UserWarning', + 'ValueError', + 'Warning', + 'ZeroDivisionError', +] + +if IS_PY2: + _safe_names.extend([ + 'basestring', + 'cmp', + 'long', + 'unichr', + 'unicode', + 'xrange', + ]) + _safe_exceptions.extend([ + 'StandardError', + ]) +else: + _safe_names.extend([ + '__build_class__', # needed to define new classes + ]) + +for name in _safe_names: + safe_builtins[name] = getattr(builtins, name) + +for name in _safe_exceptions: + safe_builtins[name] = getattr(builtins, name) - safe_builtins[name] = __builtins__[name] # Wrappers provided by this module: # delattr @@ -81,14 +184,10 @@ # object # property # reload -# slice # staticmethod # super # type -for name in dir(exceptions): - if name[0] != "_": - safe_builtins[name] = getattr(exceptions, name) def _write_wrapper(): # Construct the write wrapper class @@ -98,44 +197,102 @@ def handler(self, *args): try: f = getattr(self.ob, secattr) except AttributeError: - raise TypeError, error_msg + raise TypeError(error_msg) f(*args) return handler - class Wrapper: + + class Wrapper(object): def __len__(self): # Required for slices with negative bounds. return len(self.ob) + def __init__(self, ob): self.__dict__['ob'] = ob - __setitem__ = _handler('__guarded_setitem__', - 'object does not support item or slice assignment') - __delitem__ = _handler('__guarded_delitem__', - 'object does not support item or slice assignment') - __setattr__ = _handler('__guarded_setattr__', - 'attribute-less object (assign or del)') - __delattr__ = _handler('__guarded_delattr__', - 'attribute-less object (assign or del)') + + __setitem__ = _handler( + '__guarded_setitem__', + 'object does not support item or slice assignment') + + __delitem__ = _handler( + '__guarded_delitem__', + 'object does not support item or slice assignment') + + __setattr__ = _handler( + '__guarded_setattr__', + 'attribute-less object (assign or del)') + + __delattr__ = _handler( + '__guarded_delattr__', + 'attribute-less object (assign or del)') return Wrapper + def _full_write_guard(): # Nested scope abuse! - # safetype and Wrapper variables are used by guard() - safetype = {dict: True, list: True}.has_key + # safetypes and Wrapper variables are used by guard() + safetypes = {dict, list} Wrapper = _write_wrapper() + def guard(ob): # Don't bother wrapping simple types, or objects that claim to # handle their own write security. - if safetype(type(ob)) or hasattr(ob, '_guarded_writes'): + if type(ob) in safetypes or hasattr(ob, '_guarded_writes'): return ob # Hand the object to the Wrapper instance, then return the instance. return Wrapper(ob) return guard + + full_write_guard = _full_write_guard() + def guarded_setattr(object, name, value): setattr(full_write_guard(object), name, value) + + safe_builtins['setattr'] = guarded_setattr + def guarded_delattr(object, name): delattr(full_write_guard(object), name) + + safe_builtins['delattr'] = guarded_delattr + + +def guarded_iter_unpack_sequence(it, spec, _getiter_): + """Protect sequence unpacking of targets in a 'for loop'. + + The target of a for loop could be a sequence. + For example "for a, b in it" + => Each object from the iterator needs guarded sequence unpacking. + """ + # The iteration itself needs to be protected as well. + for ob in _getiter_(it): + yield guarded_unpack_sequence(ob, spec, _getiter_) + + +def guarded_unpack_sequence(it, spec, _getiter_): + """Protect nested sequence unpacking. + + Protect the unpacking of 'it' by wrapping it with '_getiter_'. + Furthermore for each child element, defined by spec, + guarded_unpack_sequence is called again. + + Have a look at transformer.py 'gen_unpack_spec' for a more detailed + explanation. + """ + # Do the guarded unpacking of the sequence. + ret = list(_getiter_(it)) + + # If the sequence is shorter then expected the interpreter will raise + # 'ValueError: need more than X value to unpack' anyway + # => No childs are unpacked => nothing to protect. + if len(ret) < spec['min_len']: + return ret + + # For all child elements do the guarded unpacking again. + for (idx, child_spec) in spec['childs']: + ret[idx] = guarded_unpack_sequence(ret[idx], child_spec, _getiter_) + + return ret diff --git a/src/RestrictedPython/Limits.py b/src/RestrictedPython/Limits.py index 952ec68..0da8aad 100644 --- a/src/RestrictedPython/Limits.py +++ b/src/RestrictedPython/Limits.py @@ -11,10 +11,9 @@ # ############################################################################## -__version__='$Revision: 1.5 $'[11:-2] - limited_builtins = {} + def limited_range(iFirst, *args): # limited range function from Martijn Pieters RANGELIMIT = 1000 @@ -25,22 +24,33 @@ def limited_range(iFirst, *args): elif len(args) == 2: iStart, iEnd, iStep = iFirst, args[0], args[1] else: - raise AttributeError, 'range() requires 1-3 int arguments' - if iStep == 0: raise ValueError, 'zero step for range()' + raise AttributeError('range() requires 1-3 int arguments') + if iStep == 0: + raise ValueError('zero step for range()') iLen = int((iEnd - iStart) / iStep) - if iLen < 0: iLen = 0 - if iLen >= RANGELIMIT: raise ValueError, 'range() too large' + if iLen < 0: + iLen = 0 + if iLen >= RANGELIMIT: + raise ValueError('range() too large') return range(iStart, iEnd, iStep) + + limited_builtins['range'] = limited_range + def limited_list(seq): if isinstance(seq, str): - raise TypeError, 'cannot convert string to list' + raise TypeError('cannot convert string to list') return list(seq) + + limited_builtins['list'] = limited_list + def limited_tuple(seq): if isinstance(seq, str): - raise TypeError, 'cannot convert string to tuple' + raise TypeError('cannot convert string to tuple') return tuple(seq) + + limited_builtins['tuple'] = limited_tuple diff --git a/src/RestrictedPython/MutatingWalker.py b/src/RestrictedPython/MutatingWalker.py index 6f62727..4aaa1d1 100644 --- a/src/RestrictedPython/MutatingWalker.py +++ b/src/RestrictedPython/MutatingWalker.py @@ -11,14 +11,24 @@ # ############################################################################## -__version__='$Revision: 1.6 $'[11:-2] +from compiler import ast + +import warnings + + +warnings.warn( + "This Module (RestrictedPython.MutatingWalker) is deprecated" + "and will be gone soon.", + category=PendingDeprecationWarning, + stacklevel=1 +) -from SelectCompiler import ast ListType = type([]) TupleType = type(()) SequenceTypes = (ListType, TupleType) + class MutatingWalker: def __init__(self, visitor): @@ -43,7 +53,7 @@ def visitSequence(self, seq): if v is not child: # Change the sequence. if type(res) is ListType: - res[idx : idx + 1] = [v] + res[idx: idx + 1] = [v] else: res = res[:idx] + (v,) + res[idx + 1:] return res @@ -70,5 +80,6 @@ def dispatchNode(self, node): self._cache[klass] = meth return meth(node, self) + def walk(tree, visitor): return MutatingWalker(visitor).dispatchNode(tree) diff --git a/src/RestrictedPython/PrintCollector.py b/src/RestrictedPython/PrintCollector.py index 86c4233..44ac972 100644 --- a/src/RestrictedPython/PrintCollector.py +++ b/src/RestrictedPython/PrintCollector.py @@ -10,14 +10,26 @@ # FOR A PARTICULAR PURPOSE # ############################################################################## +from __future__ import print_function -__version__='$Revision: 1.4 $'[11:-2] -class PrintCollector: - '''Collect written text, and return it when called.''' - def __init__(self): +class PrintCollector(object): + """Collect written text, and return it when called.""" + + def __init__(self, _getattr_=None): self.txt = [] + self._getattr_ = _getattr_ + def write(self, text): self.txt.append(text) + def __call__(self): return ''.join(self.txt) + + def _call_print(self, *objects, **kwargs): + if kwargs.get('file', None) is None: + kwargs['file'] = self + else: + self._getattr_(kwargs['file'], 'write') + + print(*objects, **kwargs) diff --git a/src/RestrictedPython/RCompile.py b/src/RestrictedPython/RCompile.py index c6fd508..12c7be3 100644 --- a/src/RestrictedPython/RCompile.py +++ b/src/RestrictedPython/RCompile.py @@ -14,14 +14,31 @@ Python standard library. """ -__version__='$Revision: 1.6 $'[11:-2] - -from compiler import ast, parse, misc, syntax, pycodegen -from compiler.pycodegen import AbstractCompileMode, Expression, \ - Interactive, Module, ModuleCodeGenerator, FunctionCodeGenerator, findOp +from compile import CompileResult +from compiler import ast +from compiler import misc +from compiler import parse +from compiler import pycodegen +from compiler import syntax +from compiler.pycodegen import AbstractCompileMode +from compiler.pycodegen import Expression +from compiler.pycodegen import findOp +from compiler.pycodegen import FunctionCodeGenerator # noqa +from compiler.pycodegen import Interactive +from compiler.pycodegen import Module +from compiler.pycodegen import ModuleCodeGenerator +from RestrictionMutator import RestrictionMutator import MutatingWalker -from RestrictionMutator import RestrictionMutator +import warnings + + +warnings.warn( + "This Module (RestrictedPython.RCompile) is deprecated" + "and will be gone soon.", + category=PendingDeprecationWarning, + stacklevel=1 +) def niceParse(source, filename, mode): @@ -41,12 +58,13 @@ def niceParse(source, filename, mode): # Some other error occurred. raise + class RestrictedCompileMode(AbstractCompileMode): """Abstract base class for hooking up custom CodeGenerator.""" # See concrete subclasses below. def __init__(self, source, filename): - if source: + if source: source = '\n'.join(source.splitlines()) + '\n' self.rm = RestrictionMutator() AbstractCompileMode.__init__(self, source, filename) @@ -58,7 +76,7 @@ def _get_tree(self): tree = self.parse() MutatingWalker.walk(tree, self.rm) if self.rm.errors: - raise SyntaxError, self.rm.errors[0] + raise SyntaxError(self.rm.errors[0]) misc.set_filename(self.filename, tree) syntax.check(tree) return tree @@ -72,9 +90,14 @@ def compile(self): def compileAndTuplize(gen): try: gen.compile() - except SyntaxError, v: - return None, (str(v),), gen.rm.warnings, gen.rm.used_names - return gen.getCode(), (), gen.rm.warnings, gen.rm.used_names + except TypeError as v: + return CompileResult( + None, (str(v),), gen.rm.warnings, gen.rm.used_names) + except SyntaxError as v: + return CompileResult( + None, (str(v),), gen.rm.warnings, gen.rm.used_names) + return CompileResult(gen.getCode(), (), gen.rm.warnings, gen.rm.used_names) + def compile_restricted_function(p, body, name, filename, globalize=None): """Compiles a restricted code object for a function. @@ -87,21 +110,60 @@ def compile_restricted_function(p, body, name, filename, globalize=None): treated as globals (code is generated as if each name in the list appeared in a global statement at the top of the function). """ + warnings.warn( + "RestrictedPython.RCompile.compile_restricted_function is deprecated" + "use RestrictedPython.compile_restricted_function instead.", + category=PendingDeprecationWarning, + stacklevel=1 + ) gen = RFunction(p, body, name, filename, globalize) return compileAndTuplize(gen) -def compile_restricted_exec(s, filename=''): + +def compile_restricted_exec(source, filename=''): """Compiles a restricted code suite.""" - gen = RModule(s, filename) + warnings.warn( + "RestrictedPython.RCompile.compile_restricted_exec is deprecated" + "use RestrictedPython.compile_restricted_exec instead.", + category=PendingDeprecationWarning, + stacklevel=1 + ) + gen = RModule(source, filename) + return compileAndTuplize(gen) + + +def compile_restricted_eval(source, filename=''): + """Compiles a restricted expression.""" + warnings.warn( + "RestrictedPython.RCompile.compile_restricted_eval is deprecated" + "use RestrictedPython.compile_restricted_eval instead.", + category=PendingDeprecationWarning, + stacklevel=1 + ) + gen = RExpression(source, filename) return compileAndTuplize(gen) -def compile_restricted_eval(s, filename=''): + +def compile_restricted_single(source, filename=''): """Compiles a restricted expression.""" - gen = RExpression(s, filename) + warnings.warn( + "RestrictedPython.RCompile.compile_restricted_single is deprecated" + "use RestrictedPython.compile_restricted_single instead.", + category=PendingDeprecationWarning, + stacklevel=1 + ) + gen = RInteractive(source, filename) return compileAndTuplize(gen) + def compile_restricted(source, filename, mode): """Replacement for the builtin compile() function.""" + warnings.warn( + "RestrictedPython.RCompile.compile_restricted is deprecated" + "use RestrictedPython.compile_restricted instead.", + category=PendingDeprecationWarning, + stacklevel=1 + ) if mode == "single": gen = RInteractive(source, filename) elif mode == "exec": @@ -114,6 +176,7 @@ def compile_restricted(source, filename, mode): gen.compile() return gen.getCode() + class RestrictedCodeGenerator: """Mixin for CodeGenerator to replace UNPACK_SEQUENCE bytecodes. @@ -167,18 +230,22 @@ def unpackSequence(self, tup): # handle unpacking for all the different compilation modes. They # are defined here (at the end) so that can refer to RestrictedCodeGenerator. + class RestrictedFunctionCodeGenerator(RestrictedCodeGenerator, pycodegen.FunctionCodeGenerator): pass + class RestrictedExpressionCodeGenerator(RestrictedCodeGenerator, pycodegen.ExpressionCodeGenerator): pass + class RestrictedInteractiveCodeGenerator(RestrictedCodeGenerator, pycodegen.InteractiveCodeGenerator): pass + class RestrictedModuleCodeGenerator(RestrictedCodeGenerator, pycodegen.ModuleCodeGenerator): @@ -197,14 +264,17 @@ class RExpression(RestrictedCompileMode, Expression): mode = "eval" CodeGeneratorClass = RestrictedExpressionCodeGenerator + class RInteractive(RestrictedCompileMode, Interactive): mode = "single" CodeGeneratorClass = RestrictedInteractiveCodeGenerator + class RModule(RestrictedCompileMode, Module): mode = "exec" CodeGeneratorClass = RestrictedModuleCodeGenerator + class RFunction(RModule): """A restricted Python function built from parts.""" @@ -232,8 +302,8 @@ def parse(self): if len(f.code.nodes) > 0: stmt1 = f.code.nodes[0] if (isinstance(stmt1, ast.Discard) and - isinstance(stmt1.expr, ast.Const) and - isinstance(stmt1.expr.value, str)): + isinstance(stmt1.expr, ast.Const) and + isinstance(stmt1.expr.value, str)): f.doc = stmt1.expr.value # The caller may specify that certain variables are globals # so that they can be referenced before a local assignment. diff --git a/src/RestrictedPython/README.txt b/src/RestrictedPython/README.rst similarity index 95% rename from src/RestrictedPython/README.txt rename to src/RestrictedPython/README.rst index 84a0f09..ddf7f1b 100644 --- a/src/RestrictedPython/README.txt +++ b/src/RestrictedPython/README.rst @@ -11,7 +11,7 @@ controlled and restricted execution of code: ... def hello_world(): ... return "Hello World!" ... ''' - >>> from RestrictedPython import compile_restricted + >>> from RestrictedPython.RCompile import compile_restricted >>> code = compile_restricted(src, '', 'exec') The resulting code can be executed using the ``exec`` built-in: @@ -33,9 +33,8 @@ and 2.7. Implementing a policy ===================== -RestrictedPython only provides the raw material for restricted -execution. To actually enforce any restrictions, you need to supply a -policy implementation by providing restricted versions of ``print``, +RestrictedPython only provides the raw material for restricted execution. +To actually enforce any restrictions, you need to supply a policy implementation by providing restricted versions of ``print``, ``getattr``, ``setattr``, ``import``, etc. These restricted implementations are hooked up by providing a set of specially named objects in the global dict that you use for execution of code. @@ -100,7 +99,7 @@ callable, from which the restricted machinery will create the object): >>> from RestrictedPython.PrintCollector import PrintCollector >>> _print_ = PrintCollector - + >>> src = ''' ... print "Hello World!" ... ''' diff --git a/src/RestrictedPython/RestrictionMutator.py b/src/RestrictedPython/RestrictionMutator.py index cd226f1..743a0cb 100644 --- a/src/RestrictedPython/RestrictionMutator.py +++ b/src/RestrictedPython/RestrictionMutator.py @@ -15,29 +15,44 @@ RestrictionMutator modifies a tree produced by compiler.transformer.Transformer, restricting and enhancing the code in various ways before sending it to pycodegen. - -$Revision: 1.13 $ """ -from SelectCompiler import ast, parse, OP_ASSIGN, OP_DELETE, OP_APPLY +from compiler import ast +from compiler.consts import OP_APPLY +from compiler.consts import OP_ASSIGN +from compiler.consts import OP_DELETE +from compiler.transformer import parse + +import warnings + + +warnings.warn( + "This Module (RestrictedPython.RestrictionMutator) is deprecated" + "and will be gone soon.", + category=PendingDeprecationWarning, + stacklevel=1 +) + # These utility functions allow us to generate AST subtrees without # line number attributes. These trees can then be inserted into other # trees without affecting line numbers shown in tracebacks, etc. def rmLineno(node): """Strip lineno attributes from a code tree.""" - if node.__dict__.has_key('lineno'): + if 'lineno' in node.__dict__: del node.lineno for child in node.getChildren(): if isinstance(child, ast.Node): rmLineno(child) + def stmtNode(txt): """Make a "clean" statement node.""" node = parse(txt).node.nodes[0] rmLineno(node) return node + # The security checks are performed by a set of six functions that # must be provided by the restricted environment. @@ -56,10 +71,12 @@ def stmtNode(txt): _printed_expr = stmtNode("_print()").expr _print_target_node = stmtNode("_print = _print_()") -class FuncInfo: + +class FuncInfo(object): print_used = False printed_used = False + class RestrictionMutator: def __init__(self): @@ -385,8 +402,8 @@ def visitAugAssign(self, node, walker): ast.Name(node.node.name), node.expr, ] - ), - ) + ), + ) newnode.lineno = node.lineno return newnode else: diff --git a/src/RestrictedPython/SelectCompiler.py b/src/RestrictedPython/SelectCompiler.py index 3243d12..debda17 100644 --- a/src/RestrictedPython/SelectCompiler.py +++ b/src/RestrictedPython/SelectCompiler.py @@ -13,14 +13,24 @@ """Compiler selector. """ -# Use the compiler from the standard library. -import compiler from compiler import ast +from compiler.consts import OP_APPLY +from compiler.consts import OP_ASSIGN +from compiler.consts import OP_DELETE from compiler.transformer import parse -from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY +from RCompile import compile_restricted +from RCompile import compile_restricted_eval +from RCompile import compile_restricted_exec +from RCompile import compile_restricted_function + +# Use the compiler from the standard library. +import compiler +import warnings + -from RCompile import \ - compile_restricted, \ - compile_restricted_function, \ - compile_restricted_exec, \ - compile_restricted_eval +warnings.warn( + "This Module (RestrictedPython.SelectCompiler) is deprecated" + "and will be gone soon.", + category=PendingDeprecationWarning, + stacklevel=1 +) diff --git a/src/RestrictedPython/Utilities.py b/src/RestrictedPython/Utilities.py index 6fdfc49..160fecd 100644 --- a/src/RestrictedPython/Utilities.py +++ b/src/RestrictedPython/Utilities.py @@ -11,22 +11,10 @@ # ############################################################################## -__version__='$Revision: 1.7 $'[11:-2] - import math import random import string -import warnings -_old_filters = warnings.filters[:] -warnings.filterwarnings('ignore', category=DeprecationWarning) -try: - try: - import sets - except ImportError: - sets = None -finally: - warnings.filters[:] = _old_filters utility_builtins = {} @@ -34,33 +22,44 @@ utility_builtins['math'] = math utility_builtins['random'] = random utility_builtins['whrandom'] = random -utility_builtins['sets'] = sets +utility_builtins['set'] = set +utility_builtins['frozenset'] = frozenset + +try: + import sets + utility_builtins['sets'] = sets +except ImportError: + pass try: import DateTime - utility_builtins['DateTime']= DateTime.DateTime + utility_builtins['DateTime'] = DateTime.DateTime except ImportError: pass def same_type(arg1, *args): - '''Compares the class or type of two or more objects.''' + """Compares the class or type of two or more objects.""" t = getattr(arg1, '__class__', type(arg1)) for arg in args: if getattr(arg, '__class__', type(arg)) is not t: return 0 return 1 + + utility_builtins['same_type'] = same_type def test(*args): length = len(args) for i in range(1, length, 2): - if args[i-1]: + if args[i - 1]: return args[i] if length % 2: return args[-1] + + utility_builtins['test'] = test @@ -98,4 +97,6 @@ def reorder(s, with_=None, without=()): del orig[key] return result + + utility_builtins['reorder'] = reorder diff --git a/src/RestrictedPython/__init__.py b/src/RestrictedPython/__init__.py index 1a55c7a..04b55c5 100644 --- a/src/RestrictedPython/__init__.py +++ b/src/RestrictedPython/__init__.py @@ -10,9 +10,40 @@ # FOR A PARTICULAR PURPOSE # ############################################################################## -''' -RestrictedPython package. -''' +"""RestrictedPython package.""" -from SelectCompiler import * -from PrintCollector import PrintCollector +# flake8: NOQA: E401 + +# This is a file to define public API in the base namespace of the package. +# use: isor:skip to supress all isort related warnings / errors, +# as this file should be logically grouped imports + + +# Old API --> Old Import Locations (Deprecated) +# from RestrictedPython.RCompile import compile_restricted +# from RestrictedPython.RCompile import compile_restricted_eval +# from RestrictedPython.RCompile import compile_restricted_exec +# from RestrictedPython.RCompile import compile_restricted_function + +# new API Style +# compile_restricted methods: +from RestrictedPython.compile import compile_restricted # isort:skip +from RestrictedPython.compile import compile_restricted_eval # isort:skip +from RestrictedPython.compile import compile_restricted_exec # isort:skip +from RestrictedPython.compile import compile_restricted_function # isort:skip +from RestrictedPython.compile import compile_restricted_single # isort:skip + +# predefined builtins +from RestrictedPython.Guards import safe_builtins # isort:skip +from RestrictedPython.Limits import limited_builtins # isort:skip +from RestrictedPython.Utilities import utility_builtins # isort:skip + +# Helper Methods +from RestrictedPython.PrintCollector import PrintCollector # isort:skip +from RestrictedPython.compile import CompileResult # isort:skip + +# Policy +from RestrictedPython.transformer import RestrictingNodeTransformer # isort:skip + +# +from RestrictedPython.Eval import RestrictionCapableEval diff --git a/src/RestrictedPython/_compat.py b/src/RestrictedPython/_compat.py new file mode 100644 index 0000000..5979ca5 --- /dev/null +++ b/src/RestrictedPython/_compat.py @@ -0,0 +1,8 @@ +import sys + + +_version = sys.version_info +IS_PY2 = _version.major == 2 +IS_PY3 = _version.major == 3 +IS_PY34_OR_GREATER = _version.major == 3 and _version.minor >= 4 +IS_PY35_OR_GREATER = _version.major == 3 and _version.minor >= 5 diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py new file mode 100644 index 0000000..095896e --- /dev/null +++ b/src/RestrictedPython/compile.py @@ -0,0 +1,219 @@ +from collections import namedtuple +from RestrictedPython._compat import IS_PY2 +from RestrictedPython.transformer import RestrictingNodeTransformer + +import ast +import warnings + + +CompileResult = namedtuple( + 'CompileResult', 'code, errors, warnings, used_names') +syntax_error_template = ( + 'Line {lineno}: {type}: {msg} in on statement: {statement}') + + +def _compile_restricted_mode( + source, + filename='', + mode="exec", + flags=0, + dont_inherit=False, + policy=RestrictingNodeTransformer): + byte_code = None + errors = [] + warnings = [] + used_names = {} + if policy is None: + # Unrestricted Source Checks + byte_code = compile(source, filename, mode=mode, flags=flags, + dont_inherit=dont_inherit) + elif issubclass(policy, RestrictingNodeTransformer): + c_ast = None + allowed_source_types = [str, ast.Module] + if IS_PY2: + allowed_source_types.append(unicode) + if not issubclass(type(source), tuple(allowed_source_types)): + raise TypeError('Not allowed source type: ' + '"{0.__class__.__name__}".'.format(source)) + c_ast = None + # workaround for pypy issue https://bitbucket.org/pypy/pypy/issues/2552 + if isinstance(source, ast.Module): + c_ast = source + else: + try: + c_ast = ast.parse(source, filename, mode) + except (TypeError, ValueError) as e: + errors.append(str(e)) + except SyntaxError as v: + errors.append(syntax_error_template.format( + lineno=v.lineno, + type=v.__class__.__name__, + msg=v.msg, + statement=v.text.strip() + )) + if c_ast: + policy(errors, warnings, used_names).visit(c_ast) + if not errors: + byte_code = compile(c_ast, filename, mode=mode # , + # flags=flags, + # dont_inherit=dont_inherit + ) + else: + raise TypeError('Unallowed policy provided for RestrictedPython') + return CompileResult(byte_code, tuple(errors), warnings, used_names) + + +def compile_restricted_exec( + source, + filename='', + flags=0, + dont_inherit=False, + policy=RestrictingNodeTransformer): + """Compile restricted for the mode `exec`.""" + return _compile_restricted_mode( + source, + filename=filename, + mode='exec', + flags=flags, + dont_inherit=dont_inherit, + policy=policy) + + +def compile_restricted_eval( + source, + filename='', + flags=0, + dont_inherit=False, + policy=RestrictingNodeTransformer): + """Compile restricted for the mode `eval`.""" + return _compile_restricted_mode( + source, + filename=filename, + mode='eval', + flags=flags, + dont_inherit=dont_inherit, + policy=policy) + + +def compile_restricted_single( + source, + filename='', + flags=0, + dont_inherit=False, + policy=RestrictingNodeTransformer): + """Compile restricted for the mode `single`.""" + return _compile_restricted_mode( + source, + filename=filename, + mode='single', + flags=flags, + dont_inherit=dont_inherit, + policy=policy) + + +def compile_restricted_function( + p, # parameters + body, + name, + filename='', + globalize=None, # List of globals (e.g. ['here', 'context', ...]) + flags=0, + dont_inherit=False, + policy=RestrictingNodeTransformer): + """Compile a restricted code object for a function. + + The globalize argument, if specified, is a list of variable names to be + treated as globals (code is generated as if each name in the list + appeared in a global statement at the top of the function). + This allows to inject global variables into the generated function that + feel like they are local variables, so the programmer who uses this doesn't + have to understand that his code is executed inside a function scope + instead of the global scope of a module. + + To actually get an executable function, you need to execute this code and + pull out the defined function out of the locals like this: + + >>> compiled = compile_restricted_function('', 'pass', 'function_name') + >>> safe_locals = {} + >>> safe_globals = {} + >>> exec(compiled.code, safe_globals, safe_locals) + >>> compiled_function = safe_locals['function_name'] + >>> result = compiled_function(*[], **{}) + + Then if you want to controll the globals for a specific call to this + function, you can regenerate the function like this: + + >>> my_call_specific_global_bindings = dict(foo='bar') + >>> safe_globals = safe_globals.copy() + >>> safe_globals.update(my_call_specific_global_bindings) + >>> import types + >>> new_function = types.FunctionType(compiled_function.__code__, \ + safe_globals, \ + '', \ + compiled_function.__defaults__ or \ + () \ + ) + >>> result = new_function(*[], **{}) + """ + # Parse the parameters and body, then combine them. + body_ast = ast.parse(body, '', 'exec') + + # The compiled code is actually executed inside a function + # (that is called when the code is called) so reading and assigning to a + # global variable like this`printed += 'foo'` would throw an + # UnboundLocalError. + # We don't want the user to need to understand this. + if globalize: + body_ast.body.insert(0, ast.Global(globalize)) + wrapper_ast = ast.parse('def masked_function_name(%s): pass' % p, + '', 'exec') + # In case the name you chose for your generated function is not a + # valid python identifier we set it after the fact + function_ast = wrapper_ast.body[0] + assert isinstance(function_ast, ast.FunctionDef) + function_ast.name = name + + wrapper_ast.body[0].body = body_ast.body + wrapper_ast = ast.fix_missing_locations(wrapper_ast) + + result = _compile_restricted_mode( + wrapper_ast, + filename=filename, + mode='exec', + flags=flags, + dont_inherit=dont_inherit, + policy=policy) + + return result + + +def compile_restricted( + source, + filename='', + mode='exec', + flags=0, + dont_inherit=False, + policy=RestrictingNodeTransformer): + """Replacement for the built-in compile() function. + + policy ... `ast.NodeTransformer` class defining the restrictions. + + """ + if mode in ['exec', 'eval', 'single', 'function']: + result = _compile_restricted_mode( + source, + filename=filename, + mode=mode, + flags=flags, + dont_inherit=dont_inherit, + policy=policy) + else: + raise TypeError('unknown mode %s', mode) + for warning in result.warnings: + warnings.warn( + warning, + SyntaxWarning + ) + if result.errors: + raise SyntaxError(result.errors) + return result.code diff --git a/src/RestrictedPython/tests/__init__.py b/src/RestrictedPython/tests/__init__.py index fe845da..d2f3ead 100644 --- a/src/RestrictedPython/tests/__init__.py +++ b/src/RestrictedPython/tests/__init__.py @@ -1 +1 @@ -'''Python package.''' +"""Python package.""" diff --git a/src/RestrictedPython/tests/before_and_after.py b/src/RestrictedPython/tests/before_and_after.py deleted file mode 100644 index 4e044c3..0000000 --- a/src/RestrictedPython/tests/before_and_after.py +++ /dev/null @@ -1,259 +0,0 @@ -############################################################################## -# -# Copyright (c) 2003 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Restricted Python transformation examples - -This module contains pairs of functions. Each pair has a before and an -after function. The after function shows the source code equivalent -of the before function after it has been modified by the restricted -compiler. - -These examples are actually used in the testRestrictions.py -checkBeforeAndAfter() unit tests, which verifies that the restricted compiler -actually produces the same output as would be output by the normal compiler -for the after function. -""" - -# getattr - -def simple_getattr_before(x): - return x.y - -def simple_getattr_after(x): - return _getattr_(x, 'y') - -# set attr - -def simple_setattr_before(): - x.y = "bar" - -def simple_setattr_after(): - _write_(x).y = "bar" - -# for loop and list comprehensions - -def simple_forloop_before(x): - for x in [1, 2, 3]: - pass - -def simple_forloop_after(x): - for x in _getiter_([1, 2, 3]): - pass - -def nested_forloop_before(x): - for x in [1, 2, 3]: - for y in "abc": - pass - -def nested_forloop_after(x): - for x in _getiter_([1, 2, 3]): - for y in _getiter_("abc"): - pass - -def simple_list_comprehension_before(): - x = [y**2 for y in whatever if y > 3] - -def simple_list_comprehension_after(): - x = [y**2 for y in _getiter_(whatever) if y > 3] - -def nested_list_comprehension_before(): - x = [x**2 + y**2 for x in whatever1 if x >= 0 - for y in whatever2 if y >= x] - -def nested_list_comprehension_after(): - x = [x**2 + y**2 for x in _getiter_(whatever1) if x >= 0 - for y in _getiter_(whatever2) if y >= x] - -# print - -def simple_print_before(): - print "foo" - -def simple_print_after(): - _print = _print_() - print >> _print, "foo" - -# getitem - -def simple_getitem_before(): - return x[0] - -def simple_getitem_after(): - return _getitem_(x, 0) - -def simple_get_tuple_key_before(): - x = y[1,2] - -def simple_get_tuple_key_after(): - x = _getitem_(y, (1,2)) - -# set item - -def simple_setitem_before(): - x[0] = "bar" - -def simple_setitem_after(): - _write_(x)[0] = "bar" - -# delitem - -def simple_delitem_before(): - del x[0] - -def simple_delitem_after(): - del _write_(x)[0] - -# a collection of function parallels to many of the above - -def function_with_print_before(): - def foo(): - print "foo" - return printed - -def function_with_print_after(): - def foo(): - _print = _print_() - print >> _print, "foo" - return _print() - -def function_with_getattr_before(): - def foo(): - return x.y - -def function_with_getattr_after(): - def foo(): - return _getattr_(x, 'y') - -def function_with_setattr_before(): - def foo(x): - x.y = "bar" - -def function_with_setattr_after(): - def foo(x): - _write_(x).y = "bar" - -def function_with_getitem_before(): - def foo(x): - return x[0] - -def function_with_getitem_after(): - def foo(x): - return _getitem_(x, 0) - -def function_with_forloop_before(): - def foo(): - for x in [1, 2, 3]: - pass - -def function_with_forloop_after(): - def foo(): - for x in _getiter_([1, 2, 3]): - pass - -# this, and all slices, won't work in these tests because the before code -# parses the slice as a slice object, while the after code can't generate a -# slice object in this way. The after code as written below -# is parsed as a call to the 'slice' name, not as a slice object. -# XXX solutions? - -#def simple_slice_before(): -# x = y[:4] - -#def simple_slice_after(): -# _getitem = _getitem_ -# x = _getitem(y, slice(None, 4)) - -# Assignment stmts in Python can be very complicated. The "no_unpack" -# test makes sure we're not doing unnecessary rewriting. -def no_unpack_before(): - x = y - x = [y] - x = y, - x = (y, (y, y), [y, (y,)], x, (x, y)) - x = y = z = (x, y, z) - -no_unpack_after = no_unpack_before # that is, should be untouched - - -# apply() variations. Native apply() is unsafe because, e.g., -# -# def f(a, b, c): -# whatever -# -# apply(f, two_element_sequence, dict_with_key_c) -# -# or (different spelling of the same thing) -# -# f(*two_element_sequence, **dict_with_key_c) -# -# makes the elements of two_element_sequence visible to f via its 'a' and -# 'b' arguments, and the dict_with_key_c['c'] value visible via its 'c' -# argument. That is, it's a devious way to extract values without going -# thru security checks. - -def star_call_before(): - foo(*a) - -def star_call_after(): - _apply_(foo, *a) - -def star_call_2_before(): - foo(0, *a) - -def star_call_2_after(): - _apply_(foo, 0, *a) - -def starstar_call_before(): - foo(**d) - -def starstar_call_after(): - _apply_(foo, **d) - -def star_and_starstar_call_before(): - foo(*a, **d) - -def star_and_starstar_call_after(): - _apply_(foo, *a, **d) - -def positional_and_star_and_starstar_call_before(): - foo(b, *a, **d) - -def positional_and_star_and_starstar_call_after(): - _apply_(foo, b, *a, **d) - -def positional_and_defaults_and_star_and_starstar_call_before(): - foo(b, x=y, w=z, *a, **d) - -def positional_and_defaults_and_star_and_starstar_call_after(): - _apply_(foo, b, x=y, w=z, *a, **d) - -def lambda_with_getattr_in_defaults_before(): - f = lambda x=y.z: x - -def lambda_with_getattr_in_defaults_after(): - f = lambda x=_getattr_(y, "z"): x - - -# augmented operators -# Note that we don't have to worry about item, attr, or slice assignment, -# as they are disallowed. Yay! - -## def inplace_id_add_before(): -## x += y+z - -## def inplace_id_add_after(): -## x = _inplacevar_('+=', x, y+z) - - - - diff --git a/src/RestrictedPython/tests/before_and_after24.py b/src/RestrictedPython/tests/before_and_after24.py deleted file mode 100644 index 8370b56..0000000 --- a/src/RestrictedPython/tests/before_and_after24.py +++ /dev/null @@ -1,39 +0,0 @@ -############################################################################## -# -# Copyright (c) 2003 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Restricted Python transformation examples - -This module contains pairs of functions. Each pair has a before and an -after function. The after function shows the source code equivalent -of the before function after it has been modified by the restricted -compiler. - -These examples are actually used in the testRestrictions.py -checkBeforeAndAfter() unit tests, which verifies that the restricted compiler -actually produces the same output as would be output by the normal compiler -for the after function. -""" - -def simple_generator_expression_before(): - x = (y**2 for y in whatever if y > 3) - -def simple_generator_expression_after(): - x = (y**2 for y in _getiter_(whatever) if y > 3) - -def nested_generator_expression_before(): - x = (x**2 + y**2 for x in whatever1 if x >= 0 - for y in whatever2 if y >= x) - -def nested_generator_expression_after(): - x = (x**2 + y**2 for x in _getiter_(whatever1) if x >= 0 - for y in _getiter_(whatever2) if y >= x) diff --git a/src/RestrictedPython/tests/before_and_after25.py b/src/RestrictedPython/tests/before_and_after25.py deleted file mode 100644 index 1a330e5..0000000 --- a/src/RestrictedPython/tests/before_and_after25.py +++ /dev/null @@ -1,32 +0,0 @@ -############################################################################## -# -# Copyright (c) 2008 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Restricted Python transformation examples - -This module contains pairs of functions. Each pair has a before and an -after function. The after function shows the source code equivalent -of the before function after it has been modified by the restricted -compiler. - -These examples are actually used in the testRestrictions.py -checkBeforeAndAfter() unit tests, which verifies that the restricted compiler -actually produces the same output as would be output by the normal compiler -for the after function. -""" - -def simple_ternary_if_before(): - x.y = y.z if y.z else y.x - -def simple_ternary_if_after(): - _write_(x).y = _getattr_(y, 'z') if _getattr_(y, 'z') else _getattr_(y, 'x') - diff --git a/src/RestrictedPython/tests/before_and_after26.py b/src/RestrictedPython/tests/before_and_after26.py deleted file mode 100644 index a1f46ec..0000000 --- a/src/RestrictedPython/tests/before_and_after26.py +++ /dev/null @@ -1,49 +0,0 @@ -############################################################################## -# -# Copyright (c) 2008 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Restricted Python transformation examples - -This module contains pairs of functions. Each pair has a before and an -after function. The after function shows the source code equivalent -of the before function after it has been modified by the restricted -compiler. - -These examples are actually used in the testRestrictions.py -checkBeforeAndAfter() unit tests, which verifies that the restricted compiler -actually produces the same output as would be output by the normal compiler -for the after function. -""" - -def simple_context_before(): - with whatever as x: - x.y = z - -def simple_context_after(): - with whatever as x: - _write_(x).y = z - -def simple_context_assign_attr_before(): - with whatever as x.y: - x.y = z - -def simple_context_assign_attr_after(): - with whatever as _write_(x).y: - _write_(x).y = z - -def simple_context_load_attr_before(): - with whatever.w as z: - x.y = z - -def simple_context_load_attr_after(): - with _getattr_(whatever, 'w') as z: - _write_(x).y = z diff --git a/src/RestrictedPython/tests/before_and_after27.py b/src/RestrictedPython/tests/before_and_after27.py deleted file mode 100644 index a22f625..0000000 --- a/src/RestrictedPython/tests/before_and_after27.py +++ /dev/null @@ -1,45 +0,0 @@ -############################################################################## -# -# Copyright (c) 2010 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Restricted Python transformation examples - -This module contains pairs of functions. Each pair has a before and an -after function. The after function shows the source code equivalent -of the before function after it has been modified by the restricted -compiler. - -These examples are actually used in the testRestrictions.py -checkBeforeAndAfter() unit tests, which verifies that the restricted compiler -actually produces the same output as would be output by the normal compiler -for the after function. -""" - -# dictionary and set comprehensions - -def simple_dict_comprehension_before(): - x = {y: y for y in whatever if y} - -def simple_dict_comprehension_after(): - x = {y: y for y in _getiter_(whatever) if y} - -def dict_comprehension_attrs_before(): - x = {y: y.q for y in whatever.z if y.q} - -def dict_comprehension_attrs_after(): - x = {y: _getattr_(y, 'q') for y in _getiter_(_getattr_(whatever, 'z')) if _getattr_(y, 'q')} - -def simple_set_comprehension_before(): - x = {y for y in whatever if y} - -def simple_set_comprehension_after(): - x = {y for y in _getiter_(whatever) if y} diff --git a/src/RestrictedPython/tests/class.py b/src/RestrictedPython/tests/class.py index b660ffd..cc86e8e 100644 --- a/src/RestrictedPython/tests/class.py +++ b/src/RestrictedPython/tests/class.py @@ -10,4 +10,4 @@ def get(self): x.set(12) x.set(x.get() + 1) if x.get() != 13: - raise AssertionError, "expected 13, got %d" % x.get() + raise AssertionError("expected 13, got %d" % x.get()) diff --git a/src/RestrictedPython/tests/lambda.py b/src/RestrictedPython/tests/lambda.py deleted file mode 100644 index 9a268b7..0000000 --- a/src/RestrictedPython/tests/lambda.py +++ /dev/null @@ -1,5 +0,0 @@ -f = lambda x, y=1: x + y -if f(2) != 3: - raise ValueError -if f(2, 2) != 4: - raise ValueError diff --git a/src/RestrictedPython/tests/restricted_module.py b/src/RestrictedPython/tests/restricted_module.py index 4800fea..e6e7cc4 100644 --- a/src/RestrictedPython/tests/restricted_module.py +++ b/src/RestrictedPython/tests/restricted_module.py @@ -1,23 +1,28 @@ import sys + def print0(): print 'Hello, world!', return printed + def print1(): print 'Hello,', print 'world!', return printed + def printStuff(): print 'a', 'b', 'c', return printed + def printToNone(): x = None print >>x, 'Hello, world!', return printed + def printLines(): # This failed before Zope 2.4.0a2 r = range(3) @@ -27,28 +32,33 @@ def printLines(): print return printed + def try_map(): - inc = lambda i: i+1 + inc = lambda i: i + 1 x = [1, 2, 3] print map(inc, x), return printed + def try_apply(): def f(x, y, z): return x + y + z print f(*(300, 20), **{'z': 1}), return printed + def try_inplace(): x = 1 x += 3 + def primes(): # Somewhat obfuscated code on purpose print filter(None,map(lambda y:y*reduce(lambda x,y:x*y!=0, map(lambda x,y=y:y%x,range(2,int(pow(y,0.5)+1))),1),range(2,20))), return printed + def allowed_read(ob): print ob.allowed print ob.s @@ -58,13 +68,14 @@ def allowed_read(ob): print len(ob) return printed + def allowed_default_args(ob): def f(a=ob.allowed, s=ob.s): return a, s def allowed_simple(): - q = {'x':'a'} + q = {'x': 'a'} q['y'] = 'b' q.update({'z': 'c'}) r = ['a'] @@ -80,62 +91,78 @@ def allowed_simple(): return q['x'] + q['y'] + q['z'] + r[0] + r[1] + r[2] + s + def allowed_write(ob): ob.writeable = 1 - #ob.writeable += 1 - [1 for ob.writeable in 1,2] + # ob.writeable += 1 + [1 for ob.writeable in 1, 2] ob['safe'] = 2 - #ob['safe'] += 2 - [1 for ob['safe'] in 1,2] + # ob['safe'] += 2 + [1 for ob['safe'] in 1, 2] + def denied_print(ob): print >> ob, 'Hello, world!', + def denied_getattr(ob): - #ob.disallowed += 1 + # ob.disallowed += 1 ob.disallowed = 1 return ob.disallowed + def denied_default_args(ob): def f(d=ob.disallowed): return d + def denied_setattr(ob): ob.allowed = -1 + def denied_setattr2(ob): - #ob.allowed += -1 + # ob.allowed += -1 ob.allowed = -1 + def denied_setattr3(ob): - [1 for ob.allowed in 1,2] + [1 for ob.allowed in 1, 2] + def denied_getitem(ob): ob[1] + def denied_getitem2(ob): - #ob[1] += 1 + # ob[1] += 1 ob[1] + def denied_setitem(ob): ob['x'] = 2 + def denied_setitem2(ob): - #ob[0] += 2 + # ob[0] += 2 ob['x'] = 2 + def denied_setitem3(ob): - [1 for ob['x'] in 1,2] + [1 for ob['x'] in 1, 2] + def denied_setslice(ob): ob[0:1] = 'a' + def denied_setslice2(ob): - #ob[0:1] += 'a' + # ob[0:1] += 'a' ob[0:1] = 'a' + def denied_setslice3(ob): - [1 for ob[0:1] in 1,2] + [1 for ob[0:1] in 1, 2] + ##def strange_attribute(): ## # If a guard has attributes with names that don't start with an @@ -146,6 +173,7 @@ def denied_setslice3(ob): def order_of_operations(): return 3 * 4 * -2 + 2 * 12 + def rot13(ss): mapping = {} orda = ord('a') @@ -165,15 +193,18 @@ def rot13(ss): res = res + mapping.get(c, c) return res + def nested_scopes_1(): # Fails if 'a' is consumed by the first function. a = 1 + def f1(): return a + def f2(): return a return f1() + f2() + class Classic: pass - diff --git a/src/RestrictedPython/tests/security_in_syntax.py b/src/RestrictedPython/tests/security_in_syntax.py deleted file mode 100644 index f69a94e..0000000 --- a/src/RestrictedPython/tests/security_in_syntax.py +++ /dev/null @@ -1,70 +0,0 @@ -# These are all supposed to raise a SyntaxError when using -# compile_restricted() but not when using compile(). -# Each function in this module is compiled using compile_restricted(). - -def overrideGuardWithFunction(): - def _getattr(o): return o - -def overrideGuardWithLambda(): - lambda o, _getattr=None: o - -def overrideGuardWithClass(): - class _getattr: - pass - -def overrideGuardWithName(): - _getattr = None - -def overrideGuardWithArgument(): - def f(_getattr=None): - pass - -def reserved_names(): - printed = '' - -def bad_name(): - __ = 12 - -def bad_attr(): - some_ob._some_attr = 15 - -def no_exec(): - exec 'q = 1' - -def no_yield(): - yield 42 - -def check_getattr_in_lambda(arg=lambda _getattr=(lambda ob, name: name): - _getattr): - 42 - -def import_as_bad_name(): - import os as _leading_underscore - -def from_import_as_bad_name(): - from x import y as _leading_underscore - -def except_using_bad_name(): - try: - foo - except NameError, _leading_underscore: - # The name of choice (say, _write) is now assigned to an exception - # object. Hard to exploit, but conceivable. - pass - -def keyword_arg_with_bad_name(): - def f(okname=1, __badname=2): - pass - -def no_augmeneted_assignment_to_sub(): - a[b] += c - -def no_augmeneted_assignment_to_attr(): - a.b += c - -def no_augmeneted_assignment_to_slice(): - a[x:y] += c - -def no_augmeneted_assignment_to_slice2(): - a[x:y:z] += c - diff --git a/src/RestrictedPython/tests/security_in_syntax26.py b/src/RestrictedPython/tests/security_in_syntax26.py deleted file mode 100644 index 8611a27..0000000 --- a/src/RestrictedPython/tests/security_in_syntax26.py +++ /dev/null @@ -1,16 +0,0 @@ -# These are all supposed to raise a SyntaxError when using -# compile_restricted() but not when using compile(). -# Each function in this module is compiled using compile_restricted(). - -def with_as_bad_name(): - with x as _leading_underscore: - pass - -def relative_import_as_bad_name(): - from .x import y as _leading_underscore - -def except_as_bad_name(): - try: - 1/0 - except Exception as _leading_underscore: - pass diff --git a/src/RestrictedPython/tests/security_in_syntax27.py b/src/RestrictedPython/tests/security_in_syntax27.py deleted file mode 100644 index f850c8c..0000000 --- a/src/RestrictedPython/tests/security_in_syntax27.py +++ /dev/null @@ -1,13 +0,0 @@ -# These are all supposed to raise a SyntaxError when using -# compile_restricted() but not when using compile(). -# Each function in this module is compiled using compile_restricted(). - -def dict_comp_bad_name(): - {y: y for _restricted_name in x} - -def set_comp_bad_name(): - {y for _restricted_name in x} - -def compound_with_bad_name(): - with a as b, c as _restricted_name: - pass diff --git a/src/RestrictedPython/tests/testCompile.py b/src/RestrictedPython/tests/testCompile.py index df82d41..859b20d 100644 --- a/src/RestrictedPython/tests/testCompile.py +++ b/src/RestrictedPython/tests/testCompile.py @@ -11,12 +11,12 @@ # FOR A PARTICULAR PURPOSE # ############################################################################## -__version__ = '$Revision$'[11:-2] - -import unittest from RestrictedPython.RCompile import niceParse + import compiler.ast +import unittest + class CompileTests(unittest.TestCase): diff --git a/src/RestrictedPython/tests/testREADME.py b/src/RestrictedPython/tests/testREADME.py index 8b8b9ad..6800c4d 100644 --- a/src/RestrictedPython/tests/testREADME.py +++ b/src/RestrictedPython/tests/testREADME.py @@ -13,12 +13,14 @@ ############################################################################## """Run tests in README.txt """ -import unittest from doctest import DocFileSuite +import unittest + + __docformat__ = "reStructuredText" + def test_suite(): return unittest.TestSuite([ - DocFileSuite('README.txt', package='RestrictedPython'), - ]) + DocFileSuite('README.rst', package='RestrictedPython'), ]) diff --git a/src/RestrictedPython/tests/testRestrictions.py b/src/RestrictedPython/tests/testRestrictions.py index 8f732ad..7527ffc 100644 --- a/src/RestrictedPython/tests/testRestrictions.py +++ b/src/RestrictedPython/tests/testRestrictions.py @@ -1,8 +1,3 @@ -import os -import re -import sys -import unittest - # Note that nothing should be imported from AccessControl, and in particular # nothing from ZopeGuards.py. Transformed code may need several wrappers # in order to run at all, and most of the production wrappers are defined @@ -10,32 +5,46 @@ # AccessControl, so we need to define throwaway wrapper implementations # here instead. -from RestrictedPython import compile_restricted, PrintCollector -from RestrictedPython.Eval import RestrictionCapableEval -from RestrictedPython.tests import restricted_module, verify -from RestrictedPython.RCompile import RModule, RFunction +from RestrictedPython import PrintCollector +from RestrictedPython.RCompile import compile_restricted +from RestrictedPython.RCompile import RFunction +from RestrictedPython.RCompile import RModule +from RestrictedPython.tests import restricted_module +from RestrictedPython.tests import verify + +import os +import re +import sys +import unittest + try: __file__ except NameError: __file__ = os.path.abspath(sys.argv[1]) -_FILEPATH = os.path.abspath( __file__ ) -_HERE = os.path.dirname( _FILEPATH ) +_FILEPATH = os.path.abspath(__file__) +_HERE = os.path.dirname(_FILEPATH) + def _getindent(line): """Returns the indentation level of the given line.""" indent = 0 for c in line: - if c == ' ': indent = indent + 1 - elif c == '\t': indent = indent + 8 - else: break + if c == ' ': + indent = indent + 1 + elif c == '\t': + indent = indent + 8 + else: + break return indent + def find_source(fn, func): """Given a func_code object, this function tries to find and return - the python source code of the function. Originally written by + the python source code of the function. + Originally written by Harm van der Heijden (H.v.d.Heijden@phys.tue.nl)""" - f = open(fn,"r") + f = open(fn, "r") for i in range(func.co_firstlineno): line = f.readline() ind = _getindent(line) @@ -46,10 +55,12 @@ def find_source(fn, func): # the following should be <= ind, but then we get # confused by multiline docstrings. Using == works most of # the time... but not always! - if _getindent(line) == ind: break + if _getindent(line) == ind: + break f.close() return fn, msg + def get_source(func): """Less silly interface to find_source""" file = func.func_globals['__file__'] @@ -59,6 +70,7 @@ def get_source(func): assert source.strip(), "Source should not be empty!" return source + def create_rmodule(): global rmodule fn = os.path.join(_HERE, 'restricted_module.py') @@ -69,8 +81,12 @@ def create_rmodule(): compile(source, fn, 'exec') # Now compile it for real code = compile_restricted(source, fn, 'exec') - rmodule = {'__builtins__':{'__import__':__import__, 'None':None, - '__name__': 'restricted_module'}} + rmodule = {'__builtins__': {'__import__': __import__, + 'None': None, + '__name__': 'restricted_module' + } + } + builtins = getattr(__builtins__, '__dict__', __builtins__) for name in ('map', 'reduce', 'int', 'pow', 'range', 'filter', 'len', 'chr', 'ord', @@ -78,10 +94,13 @@ def create_rmodule(): rmodule[name] = builtins[name] exec code in rmodule -class AccessDenied (Exception): pass + +class AccessDenied (Exception): + pass DisallowedObject = [] + class RestrictedObject: disallowed = DisallowedObject allowed = 1 @@ -127,6 +146,8 @@ def guarded_getattr(ob, name): return v SliceType = type(slice(0)) + + def guarded_getitem(ob, index): if type(index) is SliceType and index.step is None: start = index.start @@ -143,9 +164,10 @@ def guarded_getitem(ob, index): raise AccessDenied return v + def minimal_import(name, _globals, _locals, names): if name != "__future__": - raise ValueError, "Only future imports are allowed" + raise ValueError("Only future imports are allowed") import __future__ return __future__ @@ -176,18 +198,23 @@ def __setslice__(self, lo, hi, value): # A wrapper for _apply_. apply_wrapper_called = [] + + def apply_wrapper(func, *args, **kws): apply_wrapper_called.append('yes') return func(*args, **kws) inplacevar_wrapper_called = {} + + def inplacevar_wrapper(op, x, y): inplacevar_wrapper_called[op] = x, y # This is really lame. But it's just a test. :) globs = {'x': x, 'y': y} - exec 'x'+op+'y' in globs + exec('x' + op + 'y', globs) return globs['x'] + class RestrictionTests(unittest.TestCase): def execFunc(self, name, *args, **kw): func = rmodule[name] @@ -208,12 +235,12 @@ def execFunc(self, name, *args, **kw): }) return func(*args, **kw) - def checkPrint(self): + def test_Print(self): for i in range(2): res = self.execFunc('print%s' % i) self.assertEqual(res, 'Hello, world!') - def checkPrintToNone(self): + def test_PrintToNone(self): try: res = self.execFunc('printToNone') except AttributeError: @@ -222,47 +249,47 @@ def checkPrintToNone(self): else: self.fail(0, res) - def checkPrintStuff(self): + def test_PrintStuff(self): res = self.execFunc('printStuff') self.assertEqual(res, 'a b c') - def checkPrintLines(self): + def test_PrintLines(self): res = self.execFunc('printLines') - self.assertEqual(res, '0 1 2\n3 4 5\n6 7 8\n') + self.assertEqual(res, '0 1 2\n3 4 5\n6 7 8\n') - def checkPrimes(self): + def test_Primes(self): res = self.execFunc('primes') self.assertEqual(res, '[2, 3, 5, 7, 11, 13, 17, 19]') - def checkAllowedSimple(self): + def test_AllowedSimple(self): res = self.execFunc('allowed_simple') self.assertEqual(res, 'abcabcabc') - def checkAllowedRead(self): + def test_AllowedRead(self): self.execFunc('allowed_read', RestrictedObject()) - def checkAllowedWrite(self): + def test_AllowedWrite(self): self.execFunc('allowed_write', RestrictedObject()) - def checkAllowedArgs(self): + def test_AllowedArgs(self): self.execFunc('allowed_default_args', RestrictedObject()) - def checkTryMap(self): + def test_TryMap(self): res = self.execFunc('try_map') self.assertEqual(res, "[2, 3, 4]") - def checkApply(self): + def test_Apply(self): del apply_wrapper_called[:] res = self.execFunc('try_apply') self.assertEqual(apply_wrapper_called, ["yes"]) self.assertEqual(res, "321") - def checkInplace(self): + def test_Inplace(self): inplacevar_wrapper_called.clear() res = self.execFunc('try_inplace') self.assertEqual(inplacevar_wrapper_called['+='], (1, 3)) - def checkDenied(self): + def test_Denied(self): for k in rmodule.keys(): if k[:6] == 'denied': try: @@ -273,60 +300,19 @@ def checkDenied(self): else: self.fail('%s() did not trip security' % k) - def checkSyntaxSecurity(self): - self._checkSyntaxSecurity('security_in_syntax.py') - if sys.version_info >= (2, 6): - self._checkSyntaxSecurity('security_in_syntax26.py') - if sys.version_info >= (2, 7): - self._checkSyntaxSecurity('security_in_syntax27.py') - - def _checkSyntaxSecurity(self, mod_name): - # Ensures that each of the functions in security_in_syntax.py - # throws a SyntaxError when using compile_restricted. - fn = os.path.join(_HERE, mod_name) - f = open(fn, 'r') - source = f.read() - f.close() - # Unrestricted compile. - code = compile(source, fn, 'exec') - m = {'__builtins__': {'__import__':minimal_import}} - exec code in m - for k, v in m.items(): - if hasattr(v, 'func_code'): - filename, source = find_source(fn, v.func_code) - # Now compile it with restrictions - try: - code = compile_restricted(source, filename, 'exec') - except SyntaxError: - # Passed the test. - pass - else: - self.fail('%s should not have compiled' % k) - - def checkOrderOfOperations(self): + def test_OrderOfOperations(self): res = self.execFunc('order_of_operations') self.assertEqual(res, 0) - def checkRot13(self): + def test_Rot13(self): res = self.execFunc('rot13', 'Zope is k00l') self.assertEqual(res, 'Mbcr vf x00y') - def checkNestedScopes1(self): + def test_NestedScopes1(self): res = self.execFunc('nested_scopes_1') self.assertEqual(res, 2) - def checkUnrestrictedEval(self): - expr = RestrictionCapableEval("{'a':[m.pop()]}['a'] + [m[0]]") - v = [12, 34] - expect = v[:] - expect.reverse() - res = expr.eval({'m':v}) - self.assertEqual(res, expect) - v = [12, 34] - res = expr(m=v) - self.assertEqual(res, expect) - - def checkStackSize(self): + def test_StackSize(self): for k, rfunc in rmodule.items(): if not k.startswith('_') and hasattr(rfunc, 'func_code'): rss = rfunc.func_code.co_stacksize @@ -336,80 +322,6 @@ def checkStackSize(self): 'should have been at least %d, but was only %d' % (k, ss, rss)) - - def checkBeforeAndAfter(self): - from RestrictedPython.RCompile import RModule - from RestrictedPython.tests import before_and_after - from compiler import parse - - defre = re.compile(r'def ([_A-Za-z0-9]+)_(after|before)\(') - - beforel = [name for name in before_and_after.__dict__ - if name.endswith("_before")] - - for name in beforel: - before = getattr(before_and_after, name) - before_src = get_source(before) - before_src = re.sub(defre, r'def \1(', before_src) - rm = RModule(before_src, '') - tree_before = rm._get_tree() - - after = getattr(before_and_after, name[:-6]+'after') - after_src = get_source(after) - after_src = re.sub(defre, r'def \1(', after_src) - tree_after = parse(after_src) - - self.assertEqual(str(tree_before), str(tree_after)) - - rm.compile() - verify.verify(rm.getCode()) - - def _checkBeforeAndAfter(self, mod): - from RestrictedPython.RCompile import RModule - from compiler import parse - - defre = re.compile(r'def ([_A-Za-z0-9]+)_(after|before)\(') - - beforel = [name for name in mod.__dict__ - if name.endswith("_before")] - - for name in beforel: - before = getattr(mod, name) - before_src = get_source(before) - before_src = re.sub(defre, r'def \1(', before_src) - rm = RModule(before_src, '') - tree_before = rm._get_tree() - - after = getattr(mod, name[:-6]+'after') - after_src = get_source(after) - after_src = re.sub(defre, r'def \1(', after_src) - tree_after = parse(after_src) - - self.assertEqual(str(tree_before), str(tree_after)) - - rm.compile() - verify.verify(rm.getCode()) - - if sys.version_info[:2] >= (2, 4): - def checkBeforeAndAfter24(self): - from RestrictedPython.tests import before_and_after24 - self._checkBeforeAndAfter(before_and_after24) - - if sys.version_info[:2] >= (2, 5): - def checkBeforeAndAfter25(self): - from RestrictedPython.tests import before_and_after25 - self._checkBeforeAndAfter(before_and_after25) - - if sys.version_info[:2] >= (2, 6): - def checkBeforeAndAfter26(self): - from RestrictedPython.tests import before_and_after26 - self._checkBeforeAndAfter(before_and_after26) - - if sys.version_info[:2] >= (2, 7): - def checkBeforeAndAfter27(self): - from RestrictedPython.tests import before_and_after27 - self._checkBeforeAndAfter(before_and_after27) - def _compile_file(self, name): path = os.path.join(_HERE, name) f = open(path, "r") @@ -420,21 +332,22 @@ def _compile_file(self, name): verify.verify(co) return co - def checkUnpackSequence(self): + def test_UnpackSequence(self): co = self._compile_file("unpack.py") calls = [] + def getiter(seq): calls.append(seq) return list(seq) globals = {"_getiter_": getiter, '_inplacevar_': inplacevar_wrapper} - exec co in globals, {} + exec(co, globals, {}) # The comparison here depends on the exact code that is # contained in unpack.py. # The test doing implicit unpacking in an "except:" clause is # a pain, because there are two levels of unpacking, and the top # level is unpacking the specific TypeError instance constructed # by the test. We have to worm around that one. - ineffable = "a TypeError instance" + ineffable = "a TypeError instance" expected = [[1, 2], (1, 2), "12", @@ -458,29 +371,31 @@ def getiter(seq): expected[i] = calls[i] self.assertEqual(calls, expected) - def checkUnpackSequenceExpression(self): + def test_UnpackSequenceExpression(self): co = compile_restricted("[x for x, y in [(1, 2)]]", "", "eval") verify.verify(co) calls = [] + def getiter(s): calls.append(s) return list(s) globals = {"_getiter_": getiter} - exec co in globals, {} - self.assertEqual(calls, [[(1,2)], (1, 2)]) + exec(co, globals, {}) + self.assertEqual(calls, [[(1, 2)], (1, 2)]) - def checkUnpackSequenceSingle(self): + def test_UnpackSequenceSingle(self): co = compile_restricted("x, y = 1, 2", "", "single") verify.verify(co) calls = [] + def getiter(s): calls.append(s) return list(s) globals = {"_getiter_": getiter} - exec co in globals, {} + exec(co, globals, {}) self.assertEqual(calls, [(1, 2)]) - def checkClass(self): + def test_Class(self): getattr_calls = [] setattr_calls = [] @@ -496,7 +411,7 @@ def test_setattr(obj): globals = {"_getattr_": test_getattr, "_write_": test_setattr, } - exec co in globals, {} + exec(co, globals, {}) # Note that the getattr calls don't correspond to the method call # order, because the x.set method is fetched before its arguments # are evaluated. @@ -504,17 +419,13 @@ def test_setattr(obj): ["set", "set", "get", "state", "get", "state"]) self.assertEqual(setattr_calls, ["MyClass", "MyClass"]) - def checkLambda(self): - co = self._compile_file("lambda.py") - exec co in {}, {} - - def checkEmpty(self): + def test_Empty(self): rf = RFunction("", "", "issue945", "empty.py", {}) rf.parse() rf2 = RFunction("", "# still empty\n\n# by", "issue945", "empty.py", {}) rf2.parse() - def checkSyntaxError(self): + def test_SyntaxError(self): err = ("def f(x, y):\n" " if x, y < 2 + 1:\n" " return x + y\n" @@ -523,10 +434,10 @@ def checkSyntaxError(self): self.assertRaises(SyntaxError, compile_restricted, err, "", "exec") - # these two tests check that source code with Windows line + # these two tests test_ that source code with Windows line # endings still works. - def checkLineEndingsRFunction(self): + def test_LineEndingsRFunction(self): from RestrictedPython.RCompile import RFunction gen = RFunction( p='', @@ -534,39 +445,40 @@ def checkLineEndingsRFunction(self): name='test', filename='', globals=(), - ) + ) gen.mode = 'exec' # if the source has any line ending other than \n by the time # parse() is called, then you'll get a syntax error. gen.parse() - def checkLineEndingsRestrictedCompileMode(self): + def test_LineEndingsRestrictedCompileMode(self): from RestrictedPython.RCompile import RestrictedCompileMode gen = RestrictedCompileMode( '# testing\r\nprint "testing"\r\nreturn printed\n', '' - ) - gen.mode='exec' + ) + gen.mode = 'exec' # if the source has any line ending other than \n by the time # parse() is called, then you'll get a syntax error. gen.parse() - def checkCollector2295(self): + def test_Collector2295(self): from RestrictedPython.RCompile import RestrictedCompileMode gen = RestrictedCompileMode( 'if False:\n pass\n# Me Grok, Say Hi', '' - ) - gen.mode='exec' + ) + gen.mode = 'exec' # if the source has any line ending other than \n by the time # parse() is called, then you'll get a syntax error. gen.parse() - + create_rmodule() + def test_suite(): - return unittest.makeSuite(RestrictionTests, 'check') + return unittest.makeSuite(RestrictionTests, 'test') -if __name__=='__main__': +if __name__ == '__main__': unittest.main(defaultTest="test_suite") diff --git a/src/RestrictedPython/tests/testUtiliities.py b/src/RestrictedPython/tests/testUtiliities.py deleted file mode 100644 index bd0b878..0000000 --- a/src/RestrictedPython/tests/testUtiliities.py +++ /dev/null @@ -1,157 +0,0 @@ -############################################################################## -# -# Copyright (c) 2009 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Run tests in README.txt -""" -import unittest - -class UtilitiesTests(unittest.TestCase): - - def test_string_in_utility_builtins(self): - import string - from RestrictedPython.Utilities import utility_builtins - self.failUnless(utility_builtins['string'] is string) - - def test_math_in_utility_builtins(self): - import math - from RestrictedPython.Utilities import utility_builtins - self.failUnless(utility_builtins['math'] is math) - - def test_whrandom_in_utility_builtins(self): - import random - from RestrictedPython.Utilities import utility_builtins - self.failUnless(utility_builtins['whrandom'] is random) - - def test_random_in_utility_builtins(self): - import random - from RestrictedPython.Utilities import utility_builtins - self.failUnless(utility_builtins['random'] is random) - - def test_sets_in_utility_builtins_if_importable(self): - import warnings - from RestrictedPython.Utilities import utility_builtins - _old_filters = warnings.filters[:] - warnings.filterwarnings('ignore', category=DeprecationWarning) - try: - try: - import sets - except ImportError: - sets = None - finally: - warnings.filters[:] = _old_filters - self.failUnless(utility_builtins['sets'] is sets) - - def test_DateTime_in_utility_builtins_if_importable(self): - try: - import DateTime - except ImportError: - pass - else: - from RestrictedPython.Utilities import utility_builtins - self.failUnless('DateTime' in utility_builtins) - - def test_same_type_in_utility_builtins(self): - from RestrictedPython.Utilities import same_type - from RestrictedPython.Utilities import utility_builtins - self.failUnless(utility_builtins['same_type'] is same_type) - - def test_test_in_utility_builtins(self): - from RestrictedPython.Utilities import test - from RestrictedPython.Utilities import utility_builtins - self.failUnless(utility_builtins['test'] is test) - - def test_reorder_in_utility_builtins(self): - from RestrictedPython.Utilities import reorder - from RestrictedPython.Utilities import utility_builtins - self.failUnless(utility_builtins['reorder'] is reorder) - - def test_sametype_only_one_arg(self): - from RestrictedPython.Utilities import same_type - self.failUnless(same_type(object())) - - def test_sametype_only_two_args_same(self): - from RestrictedPython.Utilities import same_type - self.failUnless(same_type(object(), object())) - - def test_sametype_only_two_args_different(self): - from RestrictedPython.Utilities import same_type - class Foo(object): - pass - self.failIf(same_type(object(), Foo())) - - def test_sametype_only_multiple_args_same(self): - from RestrictedPython.Utilities import same_type - self.failUnless(same_type(object(), object(), object(), object())) - - def test_sametype_only_multipe_args_one_different(self): - from RestrictedPython.Utilities import same_type - class Foo(object): - pass - self.failIf(same_type(object(), object(), Foo())) - - def test_test_single_value_true(self): - from RestrictedPython.Utilities import test - self.failUnless(test(True)) - - def test_test_single_value_False(self): - from RestrictedPython.Utilities import test - self.failIf(test(False)) - - def test_test_even_values_first_true(self): - from RestrictedPython.Utilities import test - self.assertEqual(test(True, 'first', True, 'second'), 'first') - - def test_test_even_values_not_first_true(self): - from RestrictedPython.Utilities import test - self.assertEqual(test(False, 'first', True, 'second'), 'second') - - def test_test_odd_values_first_true(self): - from RestrictedPython.Utilities import test - self.assertEqual(test(True, 'first', True, 'second', False), 'first') - - def test_test_odd_values_not_first_true(self): - from RestrictedPython.Utilities import test - self.assertEqual(test(False, 'first', True, 'second', False), 'second') - - def test_test_odd_values_last_true(self): - from RestrictedPython.Utilities import test - self.assertEqual(test(False, 'first', False, 'second', 'third'), - 'third') - - def test_test_odd_values_last_false(self): - from RestrictedPython.Utilities import test - self.assertEqual(test(False, 'first', False, 'second', False), False) - - def test_reorder_with__None(self): - from RestrictedPython.Utilities import reorder - before = ['a', 'b', 'c', 'd', 'e'] - without = ['a', 'c', 'e'] - after = reorder(before, without=without) - self.assertEqual(after, [('b', 'b'), ('d', 'd')]) - - def test_reorder_with__not_None(self): - from RestrictedPython.Utilities import reorder - before = ['a', 'b', 'c', 'd', 'e'] - with_ = ['a', 'd'] - without = ['a', 'c', 'e'] - after = reorder(before, with_=with_, without=without) - self.assertEqual(after, [('d', 'd')]) - -def test_suite(): - return unittest.TestSuite(( - unittest.makeSuite(UtilitiesTests), - )) - -if __name__ == '__main__': - unittest.main(defaultTest='test_suite') - diff --git a/src/RestrictedPython/tests/unpack.py b/src/RestrictedPython/tests/unpack.py index 131c482..dd57fa3 100644 --- a/src/RestrictedPython/tests/unpack.py +++ b/src/RestrictedPython/tests/unpack.py @@ -1,13 +1,15 @@ # A series of short tests for unpacking sequences. + def u1(L): x, y = L assert x == 1 assert y == 2 -u1([1,2]) +u1([1, 2]) u1((1, 2)) + def u1a(L): x, y = L assert x == '1' @@ -20,7 +22,8 @@ def u1a(L): except ValueError: pass else: - raise AssertionError, "expected 'unpack list of wrong size'" + raise AssertionError("expected 'unpack list of wrong size'") + def u2(L): x, (a, b), y = L @@ -37,7 +40,8 @@ def u2(L): except TypeError: pass else: - raise AssertionError, "expected 'iteration over non-sequence'" + raise AssertionError("expected 'iteration over non-sequence'") + def u3((x, y)): assert x == 'a' @@ -46,12 +50,15 @@ def u3((x, y)): u3(('a', 'b')) + def u4(x): (a, b), c = d, (e, f) = x assert a == 1 and b == 2 and c == (3, 4) assert d == (1, 2) and e == 3 and f == 4 -u4( ((1, 2), (3, 4)) ) + +u4(((1, 2), (3, 4))) + def u5(x): try: @@ -64,6 +71,7 @@ def u5(x): u5([42, 666]) + def u6(x): expected = 0 for i, j in x: @@ -74,8 +82,10 @@ def u6(x): u6([[0, 1], [2, 3], [4, 5]]) + def u7(x): stuff = [i + j for toplevel, in x for i, j in toplevel] assert stuff == [3, 7] -u7( ([[[1, 2]]], [[[3, 4]]]) ) + +u7(([[[1, 2]]], [[[3, 4]]])) diff --git a/src/RestrictedPython/tests/verify.py b/src/RestrictedPython/tests/verify.py index 9c76176..c3122b4 100644 --- a/src/RestrictedPython/tests/verify.py +++ b/src/RestrictedPython/tests/verify.py @@ -23,6 +23,8 @@ import dis import types +import warnings + def verify(code): """Verify all code objects reachable from code. @@ -30,11 +32,19 @@ def verify(code): In particular, traverse into contained code objects in the co_consts table. """ + warnings.warn( + "RestrictedPython.test.verify is deprecated and will be gone soon." + "verify() tests on byte code level, which did not make sense" + "with new implementation which is Python Implementation independend.", + category=PendingDeprecationWarning, + stacklevel=1 + ) verifycode(code) for ob in code.co_consts: if isinstance(ob, types.CodeType): verify(ob) + def verifycode(code): try: _verifycode(code) @@ -42,6 +52,7 @@ def verifycode(code): dis.dis(code) raise + def _verifycode(code): line = code.co_firstlineno # keep a window of the last three opcodes, with the most recent first @@ -64,7 +75,7 @@ def _verifycode(code): with_context = (with_context[0], op) elif not ((op.arg == "__enter__" and window[0].opname == "ROT_TWO" and - window[1].opname == "DUP_TOP") or + window[1].opname == "DUP_TOP") or (op.arg == "append" and window[0].opname == "DUP_TOP" and window[1].opname == "BUILD_LIST")): @@ -118,15 +129,16 @@ def _verifycode(code): raise ValueError("direct attribute access %s: %s, %s:%d" % (op.opname, op.arg, code.co_filename, line)) + class Op(object): __slots__ = ( - "opname", # string, name of the opcode - "argcode", # int, the number of the argument - "arg", # any, the object, name, or value of argcode - "line", # int, line number or None - "target", # boolean, is this op the target of a jump - "pos", # int, offset in the bytecode - ) + "opname", # string, name of the opcode + "argcode", # int, the number of the argument + "arg", # any, the object, name, or value of argcode + "line", # int, line number or None + "target", # boolean, is this op the target of a jump + "pos", # int, offset in the bytecode + ) def __init__(self, opcode, pos): self.opname = dis.opname[opcode] @@ -135,6 +147,7 @@ def __init__(self, opcode, pos): self.target = False self.pos = pos + def disassemble(co, lasti=-1): code = co.co_code labels = dis.findlabels(code) @@ -152,7 +165,7 @@ def disassemble(co, lasti=-1): if i in labels: o.target = True if op > dis.HAVE_ARGUMENT: - arg = ord(code[i]) + ord(code[i+1]) * 256 + extended_arg + arg = ord(code[i]) + ord(code[i + 1]) * 256 + extended_arg extended_arg = 0 i += 2 if op == dis.EXTENDED_ARG: @@ -172,6 +185,7 @@ def disassemble(co, lasti=-1): o.arg = free[arg] yield o + # findlinestarts is copied from Python 2.4's dis module. The code # didn't exist in 2.3, but it would be painful to code disassemble() # without it. diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py new file mode 100644 index 0000000..bef1d6a --- /dev/null +++ b/src/RestrictedPython/transformer.py @@ -0,0 +1,1407 @@ +############################################################################## +# +# Copyright (c) 2002 Zope Foundation and Contributors. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE +# +############################################################################## +""" +transformer module: + +uses Python standard library ast module and its containing classes to transform +the parsed python code to create a modified AST for a byte code generation. +""" + +# This package should follow the Plone Sytleguide for Python, +# which differ from PEP8: +# http://docs.plone.org/develop/styleguide/python.html + + +from ._compat import IS_PY2 +from ._compat import IS_PY3 +from ._compat import IS_PY34_OR_GREATER +from ._compat import IS_PY35_OR_GREATER + +import ast +import contextlib +import textwrap + + +# For AugAssign the operator must be converted to a string. +IOPERATOR_TO_STR = { + # Shared by python2 and python3 + ast.Add: '+=', + ast.Sub: '-=', + ast.Mult: '*=', + ast.Div: '/=', + ast.Mod: '%=', + ast.Pow: '**=', + ast.LShift: '<<=', + ast.RShift: '>>=', + ast.BitOr: '|=', + ast.BitXor: '^=', + ast.BitAnd: '&=', + ast.FloorDiv: '//=' +} + +if IS_PY35_OR_GREATER: + IOPERATOR_TO_STR[ast.MatMult] = '@=' + + +# When new ast nodes are generated they have no 'lineno' and 'col_offset'. +# This function copies these two fields from the incoming node +def copy_locations(new_node, old_node): + assert 'lineno' in new_node._attributes + new_node.lineno = old_node.lineno + + assert 'col_offset' in new_node._attributes + new_node.col_offset = old_node.col_offset + + ast.fix_missing_locations(new_node) + + +class PrintInfo(object): + def __init__(self): + self.print_used = False + self.printed_used = False + + @contextlib.contextmanager + def new_print_scope(self): + old_print_used = self.print_used + old_printed_used = self.printed_used + + self.print_used = False + self.printed_used = False + + try: + yield + finally: + self.print_used = old_print_used + self.printed_used = old_printed_used + + +class RestrictingNodeTransformer(ast.NodeTransformer): + + def __init__(self, errors=None, warnings=None, used_names=None): + super(RestrictingNodeTransformer, self).__init__() + self.errors = [] if errors is None else errors + self.warnings = [] if warnings is None else warnings + + # All the variables used by the incoming source. + # Internal names/variables, like the ones from 'gen_tmp_name', don't + # have to be added. + # 'used_names' is for example needed by 'RestrictionCapableEval' to + # know wich names it has to supply when calling the final code. + self.used_names = {} if used_names is None else used_names + + # Global counter to construct temporary variable names. + self._tmp_idx = 0 + + self.print_info = PrintInfo() + + def gen_tmp_name(self): + # 'check_name' ensures that no variable is prefixed with '_'. + # => Its safe to use '_tmp..' as a temporary variable. + name = '_tmp%i' % self._tmp_idx + self._tmp_idx += 1 + return name + + def error(self, node, info): + """Record a security error discovered during transformation.""" + lineno = getattr(node, 'lineno', None) + self.errors.append( + 'Line {lineno}: {info}'.format(lineno=lineno, info=info)) + + def warn(self, node, info): + """Record a security error discovered during transformation.""" + lineno = getattr(node, 'lineno', None) + self.warnings.append( + 'Line {lineno}: {info}'.format(lineno=lineno, info=info)) + + def guard_iter(self, node): + """ + Converts: + for x in expr + to + for x in _getiter_(expr) + + Also used for + * list comprehensions + * dict comprehensions + * set comprehensions + * generator expresions + """ + node = self.node_contents_visit(node) + + if isinstance(node.target, ast.Tuple): + spec = self.gen_unpack_spec(node.target) + new_iter = ast.Call( + func=ast.Name('_iter_unpack_sequence_', ast.Load()), + args=[node.iter, spec, ast.Name('_getiter_', ast.Load())], + keywords=[]) + else: + new_iter = ast.Call( + func=ast.Name("_getiter_", ast.Load()), + args=[node.iter], + keywords=[]) + + copy_locations(new_iter, node.iter) + node.iter = new_iter + return node + + def is_starred(self, ob): + if IS_PY3: + return isinstance(ob, ast.Starred) + else: + return False + + def gen_unpack_spec(self, tpl): + """Generate a specification for 'guarded_unpack_sequence'. + + This spec is used to protect sequence unpacking. + The primary goal of this spec is to tell which elements in a sequence + are sequences again. These 'child' sequences have to be protected + again. + + For example there is a sequence like this: + (a, (b, c), (d, (e, f))) = g + + On a higher level the spec says: + - There is a sequence of len 3 + - The element at index 1 is a sequence again with len 2 + - The element at index 2 is a sequence again with len 2 + - The element at index 1 in this subsequence is a sequence again + with len 2 + + With this spec 'guarded_unpack_sequence' does something like this for + protection (len checks are omitted): + + t = list(_getiter_(g)) + t[1] = list(_getiter_(t[1])) + t[2] = list(_getiter_(t[2])) + t[2][1] = list(_getiter_(t[2][1])) + return t + + The 'real' spec for the case above is then: + spec = { + 'min_len': 3, + 'childs': ( + (1, {'min_len': 2, 'childs': ()}), + (2, { + 'min_len': 2, + 'childs': ( + (1, {'min_len': 2, 'childs': ()}) + ) + } + ) + ) + } + + So finally the assignment above is converted into: + (a, (b, c), (d, (e, f))) = guarded_unpack_sequence(g, spec) + """ + spec = ast.Dict(keys=[], values=[]) + + spec.keys.append(ast.Str('childs')) + spec.values.append(ast.Tuple([], ast.Load())) + + # starred elements in a sequence do not contribute into the min_len. + # For example a, b, *c = g + # g must have at least 2 elements, not 3. 'c' is empyt if g has only 2. + min_len = len([ob for ob in tpl.elts if not self.is_starred(ob)]) + offset = 0 + + for idx, val in enumerate(tpl.elts): + # After a starred element specify the child index from the back. + # Since it is unknown how many elements from the sequence are + # consumed by the starred element. + # For example a, *b, (c, d) = g + # Then (c, d) has the index '-1' + if self.is_starred(val): + offset = min_len + 1 + + elif isinstance(val, ast.Tuple): + el = ast.Tuple([], ast.Load()) + el.elts.append(ast.Num(idx - offset)) + el.elts.append(self.gen_unpack_spec(val)) + spec.values[0].elts.append(el) + + spec.keys.append(ast.Str('min_len')) + spec.values.append(ast.Num(min_len)) + + return spec + + def protect_unpack_sequence(self, target, value): + spec = self.gen_unpack_spec(target) + return ast.Call( + func=ast.Name('_unpack_sequence_', ast.Load()), + args=[value, spec, ast.Name('_getiter_', ast.Load())], + keywords=[]) + + def gen_unpack_wrapper(self, node, target, ctx='store'): + """Helper function to protect tuple unpacks. + + node: used to copy the locations for the new nodes. + target: is the tuple which must be protected. + ctx: Defines the context of the returned temporary node. + + It returns a tuple with two element. + + Element 1: Is a temporary name node which must be used to + replace the target. + The context (store, param) is defined + by the 'ctx' parameter.. + + Element 2: Is a try .. finally where the body performs the + protected tuple unpack of the temporary variable + into the original target. + """ + + # Generate a tmp name to replace the tuple with. + tmp_name = self.gen_tmp_name() + + # Generates an expressions which protects the unpack. + # converter looks like 'wrapper(tmp_name)'. + # 'wrapper' takes care to protect sequence unpacking with _getiter_. + converter = self.protect_unpack_sequence( + target, + ast.Name(tmp_name, ast.Load())) + + # Assign the expression to the original names. + # Cleanup the temporary variable. + # Generates: + # try: + # # converter is 'wrapper(tmp_name)' + # arg = converter + # finally: + # del tmp_arg + try_body = [ast.Assign(targets=[target], value=converter)] + finalbody = [self.gen_del_stmt(tmp_name)] + + if IS_PY2: + cleanup = ast.TryFinally(body=try_body, finalbody=finalbody) + else: + cleanup = ast.Try( + body=try_body, finalbody=finalbody, handlers=[], orelse=[]) + + if ctx == 'store': + ctx = ast.Store() + elif ctx == 'param': + ctx = ast.Param() + else: + raise Exception('Unsupported context type.') + + # This node is used to catch the tuple in a tmp variable. + tmp_target = ast.Name(tmp_name, ctx) + + copy_locations(tmp_target, node) + copy_locations(cleanup, node) + + return (tmp_target, cleanup) + + def gen_none_node(self): + if IS_PY34_OR_GREATER: + return ast.NameConstant(value=None) + else: + return ast.Name(id='None', ctx=ast.Load()) + + def gen_lambda(self, args, body): + return ast.Lambda( + args=ast.arguments( + args=args, vararg=None, kwarg=None, defaults=[]), + body=body) + + def gen_del_stmt(self, name_to_del): + return ast.Delete(targets=[ast.Name(name_to_del, ast.Del())]) + + def transform_slice(self, slice_): + """Transform slices into function parameters. + + ast.Slice nodes are only allowed within a ast.Subscript node. + To use a slice as an argument of ast.Call it has to be converted. + Conversion is done by calling the 'slice' function from builtins + """ + + if isinstance(slice_, ast.Index): + return slice_.value + + elif isinstance(slice_, ast.Slice): + # Create a python slice object. + args = [] + + if slice_.lower: + args.append(slice_.lower) + else: + args.append(self.gen_none_node()) + + if slice_.upper: + args.append(slice_.upper) + else: + args.append(self.gen_none_node()) + + if slice_.step: + args.append(slice_.step) + else: + args.append(self.gen_none_node()) + + return ast.Call( + func=ast.Name('slice', ast.Load()), + args=args, + keywords=[]) + + elif isinstance(slice_, ast.ExtSlice): + dims = ast.Tuple([], ast.Load()) + for item in slice_.dims: + dims.elts.append(self.transform_slice(item)) + return dims + + else: + raise Exception("Unknown slice type: {0}".format(slice_)) + + def check_name(self, node, name): + if name is None: + return + + if name.startswith('_') and name != '_': + self.error( + node, + '"{name}" is an invalid variable name because it ' + 'starts with "_"'.format(name=name)) + + elif name.endswith('__roles__'): + self.error(node, '"%s" is an invalid variable name because ' + 'it ends with "__roles__".' % name) + + elif name == "printed": + self.error(node, '"printed" is a reserved name.') + + elif name == 'print': + # Assignments to 'print' would lead to funny results. + self.error(node, '"print" is a reserved name.') + + def check_function_argument_names(self, node): + # In python3 arguments are always identifiers. + # In python2 the 'Python.asdl' specifies expressions, but + # the python grammer allows only identifiers or a tuple of + # identifiers. If its a tuple 'tuple parameter unpacking' is used, + # which is gone in python3. + # See https://www.python.org/dev/peps/pep-3113/ + + if IS_PY2: + # Needed to handle nested 'tuple parameter unpacking'. + # For example 'def foo((a, b, (c, (d, e)))): pass' + to_check = list(node.args.args) + while to_check: + item = to_check.pop() + if isinstance(item, ast.Tuple): + to_check.extend(item.elts) + else: + self.check_name(node, item.id) + + self.check_name(node, node.args.vararg) + self.check_name(node, node.args.kwarg) + + else: + for arg in node.args.args: + self.check_name(node, arg.arg) + + if node.args.vararg: + self.check_name(node, node.args.vararg.arg) + + if node.args.kwarg: + self.check_name(node, node.args.kwarg.arg) + + for arg in node.args.kwonlyargs: + self.check_name(node, arg.arg) + + def check_import_names(self, node): + """Check the names being imported. + + This is a protection against rebinding dunder names like + _getitem_, _write_ via imports. + + => 'from _a import x' is ok, because '_a' is not added to the scope. + """ + for alias in node.names: + self.check_name(node, alias.name) + if alias.asname: + self.check_name(node, alias.asname) + + return self.node_contents_visit(node) + + def inject_print_collector(self, node, position=0): + print_used = self.print_info.print_used + printed_used = self.print_info.printed_used + + if print_used or printed_used: + # Add '_print = _print_(_getattr_)' add the top of a + # function/module. + _print = ast.Assign( + targets=[ast.Name('_print', ast.Store())], + value=ast.Call( + func=ast.Name("_print_", ast.Load()), + args=[ast.Name("_getattr_", ast.Load())], + keywords=[])) + + if isinstance(node, ast.Module): + _print.lineno = position + _print.col_offset = position + ast.fix_missing_locations(_print) + else: + copy_locations(_print, node) + + node.body.insert(position, _print) + + if not printed_used: + self.warn(node, "Prints, but never reads 'printed' variable.") + + elif not print_used: + self.warn(node, "Doesn't print, but reads 'printed' variable.") + + def gen_attr_check(self, node, attr_name): + """Check if 'attr_name' is allowed on the object in node. + + It generates (_getattr_(node, attr_name) and node). + """ + + call_getattr = ast.Call( + func=ast.Name('_getattr_', ast.Load()), + args=[node, ast.Str(attr_name)], + keywords=[]) + + return ast.BoolOp(op=ast.And(), values=[call_getattr, node]) + + # Special Functions for an ast.NodeTransformer + + def generic_visit(self, node): + """Reject ast nodes which do not have a corresponding `visit_` method. + + This is needed to prevent new ast nodes from new Python versions to be + trusted before any security review. + + To access `generic_visit` on the super class use `node_contents_visit`. + """ + # TODO: To be discussed - For whom that info is relevant + # import warnings + # warnings.warn( + # '{o.__class__.__name__}' + # ' statement is not known to RestrictedPython'.format(node), + # SyntaxWarning + # ) + self.warn( + node, + '{0.__class__.__name__}' + ' statement is not known to RestrictedPython'.format(node) + ) + self.not_allowed(node) + + def not_allowed(self, node): + self.error( + node, + '{0.__class__.__name__} statements are not allowed.'.format(node)) + + def node_contents_visit(self, node): + """Visit the contents of a node.""" + return super(RestrictingNodeTransformer, self).generic_visit(node) + + # ast for Literals + + def visit_Num(self, node): + """Allow integer numbers without restrictions.""" + return self.node_contents_visit(node) + + def visit_Str(self, node): + """Allow string literals without restrictions.""" + return self.node_contents_visit(node) + + def visit_Bytes(self, node): + """Allow bytes literals without restrictions. + + Bytes is Python 3 only. + """ + return self.node_contents_visit(node) + + def visit_List(self, node): + """Allow list literals without restrictions.""" + return self.node_contents_visit(node) + + def visit_Tuple(self, node): + """Allow tuple literals without restrictions.""" + return self.node_contents_visit(node) + + def visit_Set(self, node): + """Allow set literals without restrictions.""" + return self.node_contents_visit(node) + + def visit_Dict(self, node): + """Allow dict literals without restrictions.""" + return self.node_contents_visit(node) + + def visit_Ellipsis(self, node): + """Deny using `...`. + + Ellipsis is exists only in Python 3. + """ + self.not_allowed(node) + + def visit_NameConstant(self, node): + """ + + """ + return self.node_contents_visit(node) + + # ast for Variables + + def visit_Name(self, node): + """Prevents access to protected names. + + Converts use of the name 'printed' to this expression: '_print()' + """ + + node = self.node_contents_visit(node) + + if isinstance(node.ctx, ast.Load): + if node.id == 'printed': + self.print_info.printed_used = True + new_node = ast.Call( + func=ast.Name("_print", ast.Load()), + args=[], + keywords=[]) + + copy_locations(new_node, node) + return new_node + + elif node.id == 'print': + self.print_info.print_used = True + new_node = ast.Attribute( + value=ast.Name('_print', ast.Load()), + attr="_call_print", + ctx=ast.Load()) + + copy_locations(new_node, node) + return new_node + + self.used_names[node.id] = True + + self.check_name(node, node.id) + return node + + def visit_Load(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_Store(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_Del(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_Starred(self, node): + """ + + """ + return self.node_contents_visit(node) + + # Expressions + + def visit_Expression(self, node): + """Allow Expression statements without restrictions. + + They are in the AST when using the `eval` compile mode. + """ + return self.node_contents_visit(node) + + def visit_Expr(self, node): + """Allow Expr statements (any expression) without restrictions.""" + return self.node_contents_visit(node) + + def visit_UnaryOp(self, node): + """ + UnaryOp (Unary Operations) is the overall element for: + * Not --> which should be allowed + * UAdd --> Positive notation of variables (e.g. +var) + * USub --> Negative notation of variables (e.g. -var) + """ + return self.node_contents_visit(node) + + def visit_UAdd(self, node): + """Allow positive notation of variables. (e.g. +var)""" + return self.node_contents_visit(node) + + def visit_USub(self, node): + """Allow negative notation of variables. (e.g. -var)""" + return self.node_contents_visit(node) + + def visit_Not(self, node): + """Allow the `not` operator.""" + return self.node_contents_visit(node) + + def visit_Invert(self, node): + """Allow `~` expressions.""" + return self.node_contents_visit(node) + + def visit_BinOp(self, node): + """Allow binary operations.""" + return self.node_contents_visit(node) + + def visit_Add(self, node): + """Allow `+` expressions.""" + return self.node_contents_visit(node) + + def visit_Sub(self, node): + """Allow `-` expressions.""" + return self.node_contents_visit(node) + + def visit_Mult(self, node): + """Allow `*` expressions.""" + return self.node_contents_visit(node) + + def visit_Div(self, node): + """Allow `/` expressions.""" + return self.node_contents_visit(node) + + def visit_FloorDiv(self, node): + """Allow `//` expressions.""" + return self.node_contents_visit(node) + + def visit_Mod(self, node): + """Allow `%` expressions.""" + return self.node_contents_visit(node) + + def visit_Pow(self, node): + """Allow `**` expressions.""" + return self.node_contents_visit(node) + + def visit_LShift(self, node): + """Allow `<<` expressions.""" + return self.node_contents_visit(node) + + def visit_RShift(self, node): + """Allow `>>` expressions.""" + return self.node_contents_visit(node) + + def visit_BitOr(self, node): + """Allow `|` expressions.""" + return self.node_contents_visit(node) + + def visit_BitXor(self, node): + """Allow `^` expressions.""" + return self.node_contents_visit(node) + + def visit_BitAnd(self, node): + """Allow `&` expressions.""" + return self.node_contents_visit(node) + + def visit_MatMult(self, node): + """Matrix multiplication (`@`) is currently not allowed. + + Matrix multiplication is a Python 3.5+ feature. + """ + self.not_allowed(node) + + def visit_BoolOp(self, node): + """Allow bool operator without restrictions.""" + return self.node_contents_visit(node) + + def visit_And(self, node): + """Allow bool operator `and` without restrictions.""" + return self.node_contents_visit(node) + + def visit_Or(self, node): + """Allow bool operator `or` without restrictions.""" + return self.node_contents_visit(node) + + def visit_Compare(self, node): + """Allow comparison expressions without restrictions.""" + return self.node_contents_visit(node) + + def visit_Eq(self, node): + """Allow == expressions.""" + return self.node_contents_visit(node) + + def visit_NotEq(self, node): + """Allow != expressions.""" + return self.node_contents_visit(node) + + def visit_Lt(self, node): + """Allow < expressions.""" + return self.node_contents_visit(node) + + def visit_LtE(self, node): + """Allow <= expressions.""" + return self.node_contents_visit(node) + + def visit_Gt(self, node): + """Allow > expressions.""" + return self.node_contents_visit(node) + + def visit_GtE(self, node): + """Allow >= expressions.""" + return self.node_contents_visit(node) + + def visit_Is(self, node): + """Allow `is` expressions.""" + return self.node_contents_visit(node) + + def visit_IsNot(self, node): + """Allow `is not` expressions.""" + return self.node_contents_visit(node) + + def visit_In(self, node): + """Allow `in` expressions.""" + return self.node_contents_visit(node) + + def visit_NotIn(self, node): + """Allow `not in` expressions.""" + return self.node_contents_visit(node) + + def visit_Call(self, node): + """Checks calls with '*args' and '**kwargs'. + + Note: The following happens only if '*args' or '**kwargs' is used. + + Transfroms 'foo()' into + _apply_(foo, ) + + The thing is that '_apply_' has only '*args', '**kwargs', so it gets + Python to collapse all the myriad ways to call functions + into one manageable from. + + From there, '_apply_()' wraps args and kws in guarded accessors, + then calls the function, returning the value. + """ + + if isinstance(node.func, ast.Name): + if node.func.id == 'exec': + self.error(node, 'Exec calls are not allowed.') + elif node.func.id == 'eval': + self.error(node, 'Eval calls are not allowed.') + + needs_wrap = False + + # In python2.7 till python3.4 '*args', '**kwargs' have dedicated + # attributes on the ast.Call node. + # In python 3.5 and greater this has changed due to the fact that + # multiple '*args' and '**kwargs' are possible. + # '*args' can be detected by 'ast.Starred' nodes. + # '**kwargs' can be deteced by 'keyword' nodes with 'arg=None'. + + if IS_PY35_OR_GREATER: + for pos_arg in node.args: + if isinstance(pos_arg, ast.Starred): + needs_wrap = True + + for keyword_arg in node.keywords: + if keyword_arg.arg is None: + needs_wrap = True + else: + if (node.starargs is not None) or (node.kwargs is not None): + needs_wrap = True + + node = self.node_contents_visit(node) + + if not needs_wrap: + return node + + node.args.insert(0, node.func) + node.func = ast.Name('_apply_', ast.Load()) + copy_locations(node.func, node.args[0]) + return node + + def visit_keyword(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_IfExp(self, node): + """Allow `if` expressions without restrictions.""" + return self.node_contents_visit(node) + + def visit_Attribute(self, node): + """Checks and mutates attribute access/assignment. + + 'a.b' becomes '_getattr_(a, "b")' + 'a.b = c' becomes '_write_(a).b = c' + 'del a.b' becomes 'del _write_(a).b' + + The _write_ function should return a security proxy. + """ + if node.attr.startswith('_') and node.attr != '_': + self.error( + node, + '"{name}" is an invalid attribute name because it starts ' + 'with "_".'.format(name=node.attr)) + + if node.attr.endswith('__roles__'): + self.error( + node, + '"{name}" is an invalid attribute name because it ends ' + 'with "__roles__".'.format(name=node.attr)) + + if isinstance(node.ctx, ast.Load): + node = self.node_contents_visit(node) + new_node = ast.Call( + func=ast.Name('_getattr_', ast.Load()), + args=[node.value, ast.Str(node.attr)], + keywords=[]) + + copy_locations(new_node, node) + return new_node + + elif isinstance(node.ctx, (ast.Store, ast.Del)): + node = self.node_contents_visit(node) + new_value = ast.Call( + func=ast.Name('_write_', ast.Load()), + args=[node.value], + keywords=[]) + + copy_locations(new_value, node.value) + node.value = new_value + return node + + else: + return self.node_contents_visit(node) + + # Subscripting + + def visit_Subscript(self, node): + """Transforms all kinds of subscripts. + + 'foo[bar]' becomes '_getitem_(foo, bar)' + 'foo[:ab]' becomes '_getitem_(foo, slice(None, ab, None))' + 'foo[ab:]' becomes '_getitem_(foo, slice(ab, None, None))' + 'foo[a:b]' becomes '_getitem_(foo, slice(a, b, None))' + 'foo[a:b:c]' becomes '_getitem_(foo, slice(a, b, c))' + 'foo[a, b:c] becomes '_getitem_(foo, (a, slice(b, c, None)))' + 'foo[a] = c' becomes '_write(foo)[a] = c' + 'del foo[a]' becomes 'del _write_(foo)[a]' + + The _write_ function should return a security proxy. + """ + node = self.node_contents_visit(node) + + # 'AugStore' and 'AugLoad' are defined in 'Python.asdl' as possible + # 'expr_context'. However, according to Python/ast.c + # they are NOT used by the implementation => No need to worry here. + # Instead ast.c creates 'AugAssign' nodes, which can be visited. + + if isinstance(node.ctx, ast.Load): + new_node = ast.Call( + func=ast.Name('_getitem_', ast.Load()), + args=[node.value, self.transform_slice(node.slice)], + keywords=[]) + + copy_locations(new_node, node) + return new_node + + elif isinstance(node.ctx, (ast.Del, ast.Store)): + new_value = ast.Call( + func=ast.Name('_write_', ast.Load()), + args=[node.value], + keywords=[]) + + copy_locations(new_value, node) + node.value = new_value + return node + + else: + return node + + def visit_Index(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_Slice(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_ExtSlice(self, node): + """ + + """ + return self.node_contents_visit(node) + + # Comprehensions + + def visit_ListComp(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_SetComp(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_GeneratorExp(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_DictComp(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_comprehension(self, node): + """ + + """ + return self.guard_iter(node) + + # Statements + + def visit_Assign(self, node): + """ + + """ + + node = self.node_contents_visit(node) + + if not any(isinstance(t, ast.Tuple) for t in node.targets): + return node + + # Handle sequence unpacking. + # For briefness this example omits cleanup of the temporary variables. + # Check 'transform_tuple_assign' how its done. + # + # - Single target (with nested support) + # (a, (b, (c, d))) = + # is converted to + # (a, t1) = _getiter_() + # (b, t2) = _getiter_(t1) + # (c, d) = _getiter_(t2) + # + # - Multi targets + # (a, b) = (c, d) = + # is converted to + # (c, d) = _getiter_() + # (a, b) = _getiter_() + # Why is this valid ? The original bytecode for this multi targets + # behaves the same way. + + # ast.NodeTransformer works with list results. + # He injects it at the right place of the node's parent statements. + new_nodes = [] + + # python fills the right most target first. + for target in reversed(node.targets): + if isinstance(target, ast.Tuple): + wrapper = ast.Assign( + targets=[target], + value=self.protect_unpack_sequence(target, node.value)) + new_nodes.append(wrapper) + else: + new_node = ast.Assign(targets=[target], value=node.value) + new_nodes.append(new_node) + + for new_node in new_nodes: + copy_locations(new_node, node) + + return new_nodes + + def visit_AugAssign(self, node): + """Forbid certain kinds of AugAssign + + According to the language reference (and ast.c) the following nodes + are are possible: + Name, Attribute, Subscript + + Note that although augmented assignment of attributes and + subscripts is disallowed, augmented assignment of names (such + as 'n += 1') is allowed. + 'n += 1' becomes 'n = _inplacevar_("+=", n, 1)' + """ + + node = self.node_contents_visit(node) + + if isinstance(node.target, ast.Attribute): + self.error( + node, + "Augmented assignment of attributes is not allowed.") + return node + + elif isinstance(node.target, ast.Subscript): + self.error( + node, + "Augmented assignment of object items " + "and slices is not allowed.") + return node + + elif isinstance(node.target, ast.Name): + new_node = ast.Assign( + targets=[node.target], + value=ast.Call( + func=ast.Name('_inplacevar_', ast.Load()), + args=[ + ast.Str(IOPERATOR_TO_STR[type(node.op)]), + ast.Name(node.target.id, ast.Load()), + node.value + ], + keywords=[])) + + copy_locations(new_node, node) + return new_node + + return node + + def visit_Print(self, node): + """Checks and mutates a print statement. + + Adds a target to all print statements. 'print foo' becomes + 'print >> _print, foo', where _print is the default print + target defined for this scope. + + Alternatively, if the untrusted code provides its own target, + we have to check the 'write' method of the target. + 'print >> ob, foo' becomes + 'print >> (_getattr_(ob, 'write') and ob), foo'. + Otherwise, it would be possible to call the write method of + templates and scripts; 'write' happens to be the name of the + method that changes them. + """ + + self.print_info.print_used = True + self.warn(node, + "Print statement is deprecated and " + "not avaliable anymore in Python 3.") + + node = self.node_contents_visit(node) + if node.dest is None: + node.dest = ast.Name('_print', ast.Load()) + else: + # Pre-validate access to the 'write' attribute. + node.dest = self.gen_attr_check(node.dest, 'write') + + copy_locations(node.dest, node) + return node + + def visit_Raise(self, node): + """Allow `raise` statements without restrictions.""" + return self.node_contents_visit(node) + + def visit_Assert(self, node): + """Allow assert statements without restrictions.""" + return self.node_contents_visit(node) + + def visit_Delete(self, node): + """Allow `del` statements without restrictions.""" + return self.node_contents_visit(node) + + def visit_Pass(self, node): + """Allow `pass` statements without restrictions.""" + return self.node_contents_visit(node) + + # Imports + + def visit_Import(self, node): + """Allow `import` statements with restrictions. + See check_import_names.""" + return self.check_import_names(node) + + def visit_ImportFrom(self, node): + """Allow `import from` statements with restrictions. + See check_import_names.""" + return self.check_import_names(node) + + def visit_alias(self, node): + """Allow `as` statements in import and import from statements.""" + return self.node_contents_visit(node) + + def visit_Exec(self, node): + """Deny the usage of the exec statement. + + Exists only in Python 2. + """ + self.not_allowed(node) + + # Control flow + + def visit_If(self, node): + """Allow `if` statements without restrictions.""" + return self.node_contents_visit(node) + + def visit_For(self, node): + """Allow `for` statements with some restrictions.""" + return self.guard_iter(node) + + def visit_While(self, node): + """Allow `while` statements.""" + return self.node_contents_visit(node) + + def visit_Break(self, node): + """Allow `break` statements without restrictions.""" + return self.node_contents_visit(node) + + def visit_Continue(self, node): + """Allow `continue` statements without restrictions.""" + return self.node_contents_visit(node) + + def visit_Try(self, node): + """Allow `try` without restrictions. + + This is Python 3 only, Python 2 uses TryExcept. + """ + return self.node_contents_visit(node) + + def visit_TryFinally(self, node): + """Allow `try ... finally` without restrictions.""" + return self.node_contents_visit(node) + + def visit_TryExcept(self, node): + """Allow `try ... except` without restrictions.""" + return self.node_contents_visit(node) + + def visit_ExceptHandler(self, node): + """Protect tuple unpacking on exception handlers. + + try: + ..... + except Exception as (a, b): + .... + + becomes + + try: + ..... + except Exception as tmp: + try: + (a, b) = _getiter_(tmp) + finally: + del tmp + """ + node = self.node_contents_visit(node) + + if IS_PY3: + self.check_name(node, node.name) + return node + + if not isinstance(node.name, ast.Tuple): + return node + + tmp_target, unpack = self.gen_unpack_wrapper(node, node.name) + + # Replace the tuple with the temporary variable. + node.name = tmp_target + + # Insert the unpack code within the body of the except clause. + node.body.insert(0, unpack) + + return node + + def visit_With(self, node): + """Protect tuple unpacking on with statements.""" + node = self.node_contents_visit(node) + + if IS_PY2: + items = [node] + else: + items = node.items + + for item in reversed(items): + if isinstance(item.optional_vars, ast.Tuple): + tmp_target, unpack = self.gen_unpack_wrapper( + node, + item.optional_vars) + + item.optional_vars = tmp_target + node.body.insert(0, unpack) + + return node + + def visit_withitem(self, node): + """Allow `with` statements (context managers) without restrictions.""" + return self.node_contents_visit(node) + + # Function and class definitions + + def visit_FunctionDef(self, node): + """Allow function definitions (`def`) with some restrictions.""" + self.check_name(node, node.name) + self.check_function_argument_names(node) + + with self.print_info.new_print_scope(): + node = self.node_contents_visit(node) + self.inject_print_collector(node) + + if IS_PY3: + return node + + # Protect 'tuple parameter unpacking' with '_getiter_'. + + unpacks = [] + for index, arg in enumerate(list(node.args.args)): + if isinstance(arg, ast.Tuple): + tmp_target, unpack = self.gen_unpack_wrapper( + node, arg, 'param') + + # Replace the tuple with a single (temporary) parameter. + node.args.args[index] = tmp_target + unpacks.append(unpack) + + # Add the unpacks at the front of the body. + # Keep the order, so that tuple one is unpacked first. + node.body[0:0] = unpacks + return node + + def visit_Lambda(self, node): + """Allow lambda with some restrictions.""" + self.check_function_argument_names(node) + + node = self.node_contents_visit(node) + + if IS_PY3: + return node + + # Check for tuple parameters which need _getiter_ protection + if not any(isinstance(arg, ast.Tuple) for arg in node.args.args): + return node + + # Wrap this lambda function with another. Via this wrapping it is + # possible to protect the 'tuple arguments' with _getiter_ + outer_params = [] + inner_args = [] + + for arg in node.args.args: + if isinstance(arg, ast.Tuple): + tmp_name = self.gen_tmp_name() + converter = self.protect_unpack_sequence( + arg, + ast.Name(tmp_name, ast.Load())) + + outer_params.append(ast.Name(tmp_name, ast.Param())) + inner_args.append(converter) + + else: + outer_params.append(arg) + inner_args.append(ast.Name(arg.id, ast.Load())) + + body = ast.Call(func=node, args=inner_args, keywords=[]) + new_node = self.gen_lambda(outer_params, body) + + if node.args.vararg: + new_node.args.vararg = node.args.vararg + body.starargs = ast.Name(node.args.vararg, ast.Load()) + + if node.args.kwarg: + new_node.args.kwarg = node.args.kwarg + body.kwargs = ast.Name(node.args.kwarg, ast.Load()) + + copy_locations(new_node, node) + return new_node + + def visit_arguments(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_arg(self, node): + """ + + """ + return self.node_contents_visit(node) + + def visit_Return(self, node): + """Allow `return` statements without restrictions.""" + return self.node_contents_visit(node) + + def visit_Yield(self, node): + """Deny `yield` unconditionally.""" + self.not_allowed(node) + + def visit_YieldFrom(self, node): + """Deny `yield from` unconditionally.""" + self.not_allowed(node) + + def visit_Global(self, node): + """Allow `global` statements without restrictions.""" + return self.node_contents_visit(node) + + def visit_Nonlocal(self, node): + """Deny `nonlocal` statements. + + This statement was introduced in Python 3. + """ + # TODO: Review if we want to allow it later + self.not_allowed(node) + + def visit_ClassDef(self, node): + """Check the name of a class definition.""" + self.check_name(node, node.name) + node = self.node_contents_visit(node) + if IS_PY2: + new_class_node = node + else: + if any(keyword.arg == 'metaclass' for keyword in node.keywords): + self.error( + node, 'The keyword argument "metaclass" is not allowed.') + CLASS_DEF = textwrap.dedent('''\ + class {0.name}(metaclass=__metaclass__): + pass + '''.format(node)) + new_class_node = ast.parse(CLASS_DEF).body[0] + new_class_node.body = node.body + new_class_node.bases = node.bases + new_class_node.decorator_list = node.decorator_list + return new_class_node + + def visit_Module(self, node): + """Add the print_collector (only if print is used) at the top.""" + node = self.node_contents_visit(node) + + # Inject the print collector after 'from __future__ import ....' + position = 0 + for position, child in enumerate(node.body): + if not isinstance(child, ast.ImportFrom): + break + + if not child.module == '__future__': + break + + self.inject_print_collector(node, position) + return node + + def visit_Param(self, node): + """Allow parameters without restrictions.""" + return self.node_contents_visit(node) + + # Async und await + + def visit_AsyncFunctionDef(self, node): + """Deny async functions.""" + self.not_allowed(node) + + def visit_Await(self, node): + """Deny async functionality.""" + self.not_allowed(node) + + def visit_AsyncFor(self, node): + """Deny async functionality.""" + self.not_allowed(node) + + def visit_AsyncWith(self, node): + """Deny async functionality.""" + self.not_allowed(node) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e93f012 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,87 @@ +from RestrictedPython._compat import IS_PY2 + +import RestrictedPython + + +def _compile(compile_func, source): + """Compile some source with a compile func.""" + result = compile_func(source) + assert result.errors == (), result.errors + assert result.code is not None + return result.code + + +def _exec(compile_func): + """Factory to create an execute function.""" + def _exec(source, glb=None): + code = _compile(compile_func, source) + if glb is None: + glb = {} + exec(code, glb) + return glb + # The next line can be dropped after the old implementation was dropped. + _exec.compile_func = compile_func + return _exec + + +def _eval(compile_func): + """Factory to create an eval function.""" + def _eval(source, glb=None): + code = _compile(compile_func, source) + if glb is None: + glb = {} + return eval(code, glb) + return _eval + + +def _single(compile_func): + """Factory to create an single function.""" + def _single(source, glb=None): + code = _compile(compile_func, source) + if glb is None: + glb = {} + exec(code, glb) + return glb + return _single + + +def _function(compile_func): + """Factory to create a function object.""" + def _function(source, glb=None): + code = _compile(compile_func, source) + if glb is None: + glb = {} + exec(code, glb) + return glb + return _function + + +# Define the arguments for @pytest.mark.parametrize to be able to test both the +# old and the new implementation to be equal: +# Compile in `exec` mode. +c_exec = ('c_exec', [RestrictedPython.compile.compile_restricted_exec]) +# Compile and execute in `exec` mode. +e_exec = ('e_exec', [_exec(RestrictedPython.compile.compile_restricted_exec)]) +# Compile in `eval` mode. +c_eval = ('c_eval', [RestrictedPython.compile.compile_restricted_eval]) +# Compile and execute in `eval` mode. +e_eval = ('e_eval', [_eval(RestrictedPython.compile.compile_restricted_eval)]) +# +c_function = ('c_function', [RestrictedPython.compile.compile_restricted_function]) # NOQA: E501 +e_function = ('e_function', [_function(RestrictedPython.compile.compile_restricted_function)]) # NOQA: E501 + +c_single = ('c_single', [RestrictedPython.compile.compile_restricted_single]) +e_single = ('e_single', [_single(RestrictedPython.compile.compile_restricted_single)]) # NOQA: E501 + + +if IS_PY2: + from RestrictedPython import RCompile + c_exec[1].append(RCompile.compile_restricted_exec) + c_eval[1].append(RCompile.compile_restricted_eval) + c_single[1].append(RCompile.compile_restricted_single) + c_function[1].append(RCompile.compile_restricted_function) + + e_exec[1].append(_exec(RCompile.compile_restricted_exec)) + e_eval[1].append(_eval(RCompile.compile_restricted_eval)) + e_single[1].append(_single(RCompile.compile_restricted_single)) + e_function[1].append(_function(RCompile.compile_restricted_function)) diff --git a/tests/builtins/test_limits.py b/tests/builtins/test_limits.py new file mode 100644 index 0000000..ac034be --- /dev/null +++ b/tests/builtins/test_limits.py @@ -0,0 +1,72 @@ +from RestrictedPython.Limits import limited_list +from RestrictedPython.Limits import limited_range +from RestrictedPython.Limits import limited_tuple + +import pytest + + +def test_limited_range_length_1(): + result = limited_range(1) + assert result == range(0, 1) + + +def test_limited_range_length_10(): + result = limited_range(10) + assert result == range(0, 10) + + +def test_limited_range_5_10(): + result = limited_range(5, 10) + assert result == range(5, 10) + + +def test_limited_range_5_10_sm1(): + result = limited_range(5, 10, -1) + assert result == range(5, 10, -1) + + +def test_limited_range_15_10_s2(): + result = limited_range(15, 10, 2) + assert result == range(15, 10, 2) + + +def test_limited_range_no_input(): + with pytest.raises(TypeError): + limited_range() + + +def test_limited_range_more_steps(): + with pytest.raises(AttributeError): + limited_range(0, 0, 0, 0) + + +def test_limited_range_zero_step(): + with pytest.raises(ValueError): + limited_range(0, 10, 0) + + +def test_limited_range_range_overflow(): + with pytest.raises(ValueError): + limited_range(0, 5000, 1) + + +def test_limited_list_valid_list_input(): + input = [1, 2, 3] + result = limited_list(input) + assert result == input + + +def test_limited_list_invalid_string_input(): + with pytest.raises(TypeError): + limited_list('input') + + +def test_limited_tuple_valid_list_input(): + input = [1, 2, 3] + result = limited_tuple(input) + assert result == tuple(input) + + +def test_limited_tuple_invalid_string_input(): + with pytest.raises(TypeError): + limited_tuple('input') diff --git a/tests/builtins/test_utilities.py b/tests/builtins/test_utilities.py new file mode 100644 index 0000000..2a83782 --- /dev/null +++ b/tests/builtins/test_utilities.py @@ -0,0 +1,161 @@ +from RestrictedPython._compat import IS_PY3 + +import pytest + + +def test_string_in_utility_builtins(): + import string + from RestrictedPython.Utilities import utility_builtins + assert utility_builtins['string'] is string + + +def test_math_in_utility_builtins(): + import math + from RestrictedPython.Utilities import utility_builtins + assert utility_builtins['math'] is math + + +def test_whrandom_in_utility_builtins(): + import random + from RestrictedPython.Utilities import utility_builtins + assert utility_builtins['whrandom'] is random + + +def test_random_in_utility_builtins(): + import random + from RestrictedPython.Utilities import utility_builtins + assert utility_builtins['random'] is random + + +def test_set_in_utility_builtins(): + from RestrictedPython.Utilities import utility_builtins + assert utility_builtins['set'] is set + + +@pytest.mark.skipif(IS_PY3, + reason='Python 3 has no longer includes the sets module.') +def test_sets_in_utility_builtins(): + from RestrictedPython.Utilities import utility_builtins + import sets + assert utility_builtins['sets'] is sets + + +def test_frozenset_in_utility_builtins(): + from RestrictedPython.Utilities import utility_builtins + assert utility_builtins['frozenset'] is frozenset + + +def test_DateTime_in_utility_builtins_if_importable(): + try: + import DateTime + except ImportError: + pass + else: + from RestrictedPython.Utilities import utility_builtins + assert DateTime.__name__ in utility_builtins + + +def test_same_type_in_utility_builtins(): + from RestrictedPython.Utilities import same_type + from RestrictedPython.Utilities import utility_builtins + assert utility_builtins['same_type'] is same_type + + +def test_test_in_utility_builtins(): + from RestrictedPython.Utilities import test + from RestrictedPython.Utilities import utility_builtins + assert utility_builtins['test'] is test + + +def test_reorder_in_utility_builtins(): + from RestrictedPython.Utilities import reorder + from RestrictedPython.Utilities import utility_builtins + assert utility_builtins['reorder'] is reorder + + +def test_sametype_only_one_arg(): + from RestrictedPython.Utilities import same_type + assert same_type(object()) + + +def test_sametype_only_two_args_same(): + from RestrictedPython.Utilities import same_type + assert same_type(object(), object()) + + +def test_sametype_only_two_args_different(): + from RestrictedPython.Utilities import same_type + + class Foo(object): + pass + assert same_type(object(), Foo()) is 0 + + +def test_sametype_only_multiple_args_same(): + from RestrictedPython.Utilities import same_type + assert same_type(object(), object(), object(), object()) + + +def test_sametype_only_multipe_args_one_different(): + from RestrictedPython.Utilities import same_type + + class Foo(object): + pass + assert same_type(object(), object(), Foo()) is 0 + + +def test_test_single_value_true(): + from RestrictedPython.Utilities import test + assert test(True) is True + + +def test_test_single_value_False(): + from RestrictedPython.Utilities import test + assert test(False) is False + + +def test_test_even_values_first_true(): + from RestrictedPython.Utilities import test + assert test(True, 'first', True, 'second') == 'first' + + +def test_test_even_values_not_first_true(): + from RestrictedPython.Utilities import test + assert test(False, 'first', True, 'second') == 'second' + + +def test_test_odd_values_first_true(): + from RestrictedPython.Utilities import test + assert test(True, 'first', True, 'second', False) == 'first' + + +def test_test_odd_values_not_first_true(): + from RestrictedPython.Utilities import test + assert test(False, 'first', True, 'second', False) == 'second' + + +def test_test_odd_values_last_true(): + from RestrictedPython.Utilities import test + assert test(False, 'first', False, 'second', 'third') == 'third' + + +def test_test_odd_values_last_false(): + from RestrictedPython.Utilities import test + assert test(False, 'first', False, 'second', False) is False + + +def test_reorder_with__None(): + from RestrictedPython.Utilities import reorder + before = ['a', 'b', 'c', 'd', 'e'] + without = ['a', 'c', 'e'] + after = reorder(before, without=without) + assert after == [('b', 'b'), ('d', 'd')] + + +def test_reorder_with__not_None(): + from RestrictedPython.Utilities import reorder + before = ['a', 'b', 'c', 'd', 'e'] + with_ = ['a', 'd'] + without = ['a', 'c', 'e'] + after = reorder(before, with_=with_, without=without) + assert after == [('d', 'd')] diff --git a/tests/test_Guards.py b/tests/test_Guards.py new file mode 100644 index 0000000..32040b1 --- /dev/null +++ b/tests/test_Guards.py @@ -0,0 +1,41 @@ +from RestrictedPython.Guards import safe_builtins +from tests import e_eval +from tests import e_exec + +import pytest + + +@pytest.mark.parametrize(*e_eval) +def test_Guards__safe_builtins__1(e_eval): + """It contains `slice()`.""" + restricted_globals = dict(__builtins__=safe_builtins) + assert e_eval('slice(1)', restricted_globals) == slice(1) + + +CLASS_SOURCE = ''' +class C: + value = None + def display(self): + return str(self.value) + +c1 = C() +c1.value = 2411 +b = c1.display() +''' + + +@pytest.mark.parametrize(*e_exec) +def test_Guards__safe_builtins__2(e_exec): + """It allows to define new classes by allowing `__build_class__`. + + `__build_class__` is only needed in Python 3. + """ + restricted_globals = dict( + __builtins__=safe_builtins, b=None, + __name__='restricted_module', + __metaclass__=type, + _write_=lambda x: x, + _getattr_=getattr) + + e_exec(CLASS_SOURCE, restricted_globals) + assert restricted_globals['b'] == '2411' diff --git a/tests/test_compile.py b/tests/test_compile.py new file mode 100644 index 0000000..46914ec --- /dev/null +++ b/tests/test_compile.py @@ -0,0 +1,214 @@ +from RestrictedPython import compile_restricted +from RestrictedPython import CompileResult +from RestrictedPython._compat import IS_PY2 +from RestrictedPython._compat import IS_PY3 +from tests import c_eval +from tests import c_exec +from tests import c_single +from tests import e_eval + +import pytest +import RestrictedPython.compile +import types + + +def test_compile__compile_restricted_invalid_code_input(): + with pytest.raises(TypeError): + compile_restricted(object(), '', 'exec') + with pytest.raises(TypeError): + compile_restricted(object(), '', 'eval') + with pytest.raises(TypeError): + compile_restricted(object(), '', 'single') + + +def test_compile__compile_restricted_invalid_policy_input(): + with pytest.raises(TypeError): + compile_restricted("pass", '', 'exec', policy=object) + + +def test_compile__compile_restricted_invalid_mode_input(): + with pytest.raises(TypeError): + compile_restricted("pass", '', 'invalid') + + +@pytest.mark.parametrize(*c_exec) +def test_compile__compile_restricted_exec__1(c_exec): + """It returns a CompileResult on success.""" + result = c_exec('a = 42') + assert result.__class__ == CompileResult + assert result.errors == () + assert result.warnings == [] + assert result.used_names == {} + glob = {} + exec(result.code, glob) + assert glob['a'] == 42 + + +@pytest.mark.parametrize(*c_exec) +def test_compile__compile_restricted_exec__2(c_exec): + """It compiles without restrictions if there is no policy.""" + if c_exec is RestrictedPython.compile.compile_restricted_exec: + # The old version does not support a custom policy + result = c_exec('_a = 42', policy=None) + assert result.errors == () + assert result.warnings == [] + assert result.used_names == {} + glob = {} + exec(result.code, glob) + assert glob['_a'] == 42 + + +@pytest.mark.parametrize(*c_exec) +def test_compile__compile_restricted_exec__3(c_exec): + """It returns a tuple of errors if the code is not allowed. + + There is no code in this case. + """ + result = c_exec('_a = 42\n_b = 43') + errors = ( + 'Line 1: "_a" is an invalid variable name because it starts with "_"', + 'Line 2: "_b" is an invalid variable name because it starts with "_"') + if c_exec is RestrictedPython.compile.compile_restricted_exec: + assert result.errors == errors + else: + # The old version did only return the first error message. + assert result.errors == (errors[0],) + assert result.warnings == [] + assert result.used_names == {} + assert result.code is None + + +@pytest.mark.parametrize(*c_exec) +def test_compile__compile_restricted_exec__4(c_exec): + """It does not return code on a SyntaxError.""" + result = c_exec('asdf|') + assert result.code is None + assert result.warnings == [] + assert result.used_names == {} + if c_exec is RestrictedPython.compile.compile_restricted_exec: + assert result.errors == ( + 'Line 1: SyntaxError: invalid syntax in on statement: asdf|',) + else: + # The old version had a less nice error message: + assert result.errors == ('invalid syntax (, line 1)',) + + +@pytest.mark.parametrize(*c_exec) +def test_compile__compile_restricted_exec__5(c_exec): + """It does not return code if the code contains a NULL byte.""" + result = c_exec('a = 5\x00') + assert result.code is None + assert result.warnings == [] + assert result.used_names == {} + if IS_PY2: + assert result.errors == ( + 'compile() expected string without null bytes',) + else: + assert result.errors == ( + 'source code string cannot contain null bytes',) + + +EXEC_STATEMENT = """\ +def no_exec(): + exec 'q = 1' +""" + + +@pytest.mark.skipif( + IS_PY2, + reason="exec statement in Python 2 is handled by RestrictedPython ") +@pytest.mark.parametrize(*c_exec) +def test_compile__compile_restricted_exec__10(c_exec): + """It is a SyntaxError to use the `exec` statement. (Python 3 only)""" + result = c_exec(EXEC_STATEMENT) + assert ( + "Line 2: SyntaxError: Missing parentheses in call to 'exec' in on " + "statement: exec 'q = 1'",) == result.errors + + +FUNCTION_DEF = """\ +def a(): + pass +""" + + +@pytest.mark.parametrize(*c_eval) +def test_compile__compile_restricted_eval__1(c_eval): + """It compiles code as an Expression. + + Function definitions are not allowed in Expressions. + """ + result = c_eval(FUNCTION_DEF) + if c_eval is RestrictedPython.compile.compile_restricted_eval: + assert result.errors == ( + 'Line 1: SyntaxError: invalid syntax in on statement: def a():',) + else: + assert result.errors == ('invalid syntax (, line 1)',) + + +@pytest.mark.parametrize(*e_eval) +def test_compile__compile_restricted_eval__2(e_eval): + """It compiles code as an Expression.""" + assert e_eval('4 * 6') == 24 + + +@pytest.mark.parametrize(*c_eval) +def test_compile__compile_restricted_eval__used_names(c_eval): + result = c_eval("a + b + func(x)") + assert result.errors == () + assert result.warnings == [] + assert result.used_names == {'a': True, 'b': True, 'x': True, 'func': True} + + +@pytest.mark.parametrize(*c_single) +def test_compile__compile_restricted_csingle(c_single): + """It compiles code as an Interactive.""" + result = c_single('4 * 6') + if c_single is RestrictedPython.compile.compile_restricted_single: + # New implementation disallows single mode + assert result.code is None + assert result.errors == ( + 'Line None: Interactive statements are not allowed.', + ) + else: # RestrictedPython.RCompile.compile_restricted_single + assert result.code is not None + assert result.errors == () + + +PRINT_EXAMPLE = """ +def a(): + print 'Hello World!' +""" + + +@pytest.mark.skipif( + IS_PY3, + reason="Print statement is gone in Python 3." + "Test Deprecation Warming in Python 2") +def test_compile_restricted(): + """This test checks compile_restricted itself if that emit Python warnings. + For actual tests for print statement see: test_print_stmt.py + """ + with pytest.warns(SyntaxWarning) as record: + result = compile_restricted(PRINT_EXAMPLE, '', 'exec') + assert isinstance(result, types.CodeType) + assert len(record) == 2 + assert record[0].message.args[0] == \ + 'Line 3: Print statement is deprecated ' \ + 'and not avaliable anymore in Python 3.' + assert record[1].message.args[0] == \ + "Line 2: Prints, but never reads 'printed' variable." + + +EVAL_EXAMPLE = """ +def a(): + eval('2 + 2') +""" + + +def test_compile_restricted_eval(): + """This test checks compile_restricted itself if that raise Python errors. + """ + with pytest.raises(SyntaxError, + message="Line 3: Eval calls are not allowed."): + compile_restricted(EVAL_EXAMPLE, '', 'exec') diff --git a/tests/test_compile_restricted_function.py b/tests/test_compile_restricted_function.py new file mode 100644 index 0000000..ca51aba --- /dev/null +++ b/tests/test_compile_restricted_function.py @@ -0,0 +1,205 @@ +from RestrictedPython import PrintCollector +from RestrictedPython import safe_builtins +from tests import c_function +from types import FunctionType + +import pytest + + +@pytest.mark.parametrize(*c_function) +def test_compile_restricted_function(c_function): + p = '' + body = """ +print("Hello World!") +return printed +""" + name = "hello_world" + global_symbols = [] + + result = c_function( + p, # parameters + body, + name, + filename='', + globalize=global_symbols + ) + + assert result.code is not None + assert result.errors == () + + safe_globals = { + '__name__': 'script', + '_getattr_': getattr, + '_print_': PrintCollector + } + safe_globals.update(safe_builtins) + safe_locals = {} + exec(result.code, safe_globals, safe_locals) + hello_world = safe_locals['hello_world'] + assert type(hello_world) == FunctionType + assert hello_world() == 'Hello World!\n' + + +@pytest.mark.parametrize(*c_function) +def test_compile_restricted_function_func_wrapped(c_function): + p = '' + body = """ +print("Hello World!") +return printed +""" + name = "hello_world" + global_symbols = [] + + result = c_function( + p, # parameters + body, + name, + filename='', + globalize=global_symbols + ) + + assert result.code is not None + assert result.errors == () + safe_globals = { + '__name__': 'script', + '_getattr_': getattr, + '_print_': PrintCollector, + } + safe_globals.update(safe_builtins) + + func = FunctionType(result.code, safe_globals) + func() + assert 'hello_world' in safe_globals + hello_world = safe_globals['hello_world'] + assert hello_world() == 'Hello World!\n' + + +@pytest.mark.parametrize(*c_function) +def test_compile_restricted_function_with_arguments(c_function): + p = 'input1, input2' + body = """ +print(input1 + input2) +return printed +""" + name = "hello_world" + global_symbols = [] + + result = c_function( + p, # parameters + body, + name, + filename='', + globalize=global_symbols + ) + + assert result.code is not None + assert result.errors == () + + safe_globals = { + '__name__': 'script', + '_getattr_': getattr, + '_print_': PrintCollector + } + safe_globals.update(safe_builtins) + safe_locals = {} + exec(result.code, safe_globals, safe_locals) + hello_world = safe_locals['hello_world'] + assert type(hello_world) == FunctionType + assert hello_world('Hello ', 'World!') == 'Hello World!\n' + + +@pytest.mark.parametrize(*c_function) +def test_compile_restricted_function_can_access_global_variables(c_function): + p = '' + body = """ +print(input) +return printed +""" + name = "hello_world" + global_symbols = ['input'] + + result = c_function( + p, # parameters + body, + name, + filename='', + globalize=global_symbols + ) + + assert result.code is not None + assert result.errors == () + + safe_globals = { + '__name__': 'script', + '_getattr_': getattr, + 'input': 'Hello World!', + '_print_': PrintCollector + } + safe_globals.update(safe_builtins) + safe_locals = {} + exec(result.code, safe_globals, safe_locals) + hello_world = safe_locals['hello_world'] + assert type(hello_world) == FunctionType + assert hello_world() == 'Hello World!\n' + + +@pytest.mark.parametrize(*c_function) +def test_compile_restricted_function_pretends_the_code_is_executed_in_a_global_scope(c_function): # NOQA: E501 + p = '' + body = """output = output + 'bar'""" + name = "hello_world" + global_symbols = ['output'] + + result = c_function( + p, # parameters + body, + name, + filename='', + globalize=global_symbols + ) + + assert result.code is not None + assert result.errors == () + + safe_globals = { + '__name__': 'script', + 'output': 'foo', + } + # safe_globals.update(safe_builtins) + safe_locals = {} + exec(result.code, safe_globals, safe_locals) + hello_world = safe_locals['hello_world'] + assert type(hello_world) == FunctionType + hello_world() + assert safe_globals['output'] == 'foobar' + + +@pytest.mark.parametrize(*c_function) +def test_compile_restricted_function_allows_invalid_python_identifiers_as_function_name(c_function): # NOQA: E501 + p = '' + body = """output = output + 'bar'""" + name = ".bar.__baz__" + global_symbols = ['output'] + + result = c_function( + p, # parameters + body, + name, + filename='', + globalize=global_symbols + ) + + assert result.code is not None + assert result.errors == () + + safe_globals = { + '__name__': 'script', + 'output': 'foo', + } + # safe_globals.update(safe_builtins) + safe_locals = {} + exec(result.code, safe_globals, safe_locals) + generated_function = tuple(safe_locals.values())[0] + assert type(generated_function) == FunctionType + generated_function() + assert safe_globals['output'] == 'foobar' diff --git a/tests/test_eval.py b/tests/test_eval.py new file mode 100644 index 0000000..f9046a4 --- /dev/null +++ b/tests/test_eval.py @@ -0,0 +1,42 @@ +from RestrictedPython.Eval import RestrictionCapableEval + +import pytest + + +exp = """ + {'a':[m.pop()]}['a'] \ + + [m[0]] +""" + + +def test_init(): + ob = RestrictionCapableEval(exp) + + assert ob.expr == "{'a':[m.pop()]}['a'] + [m[0]]" + assert ob.used == ('m', ) + assert ob.ucode is not None + assert ob.rcode is None + + +def test_init_with_syntax_error(): + with pytest.raises(SyntaxError): + RestrictionCapableEval("if:") + + +def test_prepRestrictedCode(): + ob = RestrictionCapableEval(exp) + ob.prepRestrictedCode() + assert ob.used == ('m', ) + assert ob.rcode is not None + + +def test_call(): + ob = RestrictionCapableEval(exp) + ret = ob(m=[1, 2]) + assert ret == [2, 1] + + +def test_eval(): + ob = RestrictionCapableEval(exp) + ret = ob.eval({'m': [1, 2]}) + assert ret == [2, 1] diff --git a/tests/test_print_function.py b/tests/test_print_function.py new file mode 100644 index 0000000..047e0fe --- /dev/null +++ b/tests/test_print_function.py @@ -0,0 +1,360 @@ +from RestrictedPython.PrintCollector import PrintCollector + +import RestrictedPython + + +# The old 'RCompile' has no clue about the print function. +compiler = RestrictedPython.compile.compile_restricted_exec + + +ALLOWED_PRINT_FUNCTION = """ +from __future__ import print_function +print ('Hello World!') +""" + +ALLOWED_PRINT_FUNCTION_WITH_END = """ +from __future__ import print_function +print ('Hello World!', end='') +""" + +ALLOWED_PRINT_FUNCTION_MULTI_ARGS = """ +from __future__ import print_function +print ('Hello World!', 'Hello Earth!') +""" + +ALLOWED_PRINT_FUNCTION_WITH_SEPARATOR = """ +from __future__ import print_function +print ('a', 'b', 'c', sep='|', end='!') +""" + +PRINT_FUNCTION_WITH_NONE_SEPARATOR = """ +from __future__ import print_function +print ('a', 'b', sep=None) +""" + + +PRINT_FUNCTION_WITH_NONE_END = """ +from __future__ import print_function +print ('a', 'b', end=None) +""" + + +PRINT_FUNCTION_WITH_NONE_FILE = """ +from __future__ import print_function +print ('a', 'b', file=None) +""" + + +def test_print_function__simple_prints(): + glb = {'_print_': PrintCollector, '_getattr_': None} + + code, errors = compiler(ALLOWED_PRINT_FUNCTION)[:2] + assert errors == () + assert code is not None + exec(code, glb) + assert glb['_print']() == 'Hello World!\n' + + code, errors = compiler(ALLOWED_PRINT_FUNCTION_WITH_END)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == 'Hello World!' + + code, errors = compiler(ALLOWED_PRINT_FUNCTION_MULTI_ARGS)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == 'Hello World! Hello Earth!\n' + + code, errors = compiler(ALLOWED_PRINT_FUNCTION_WITH_SEPARATOR)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == "a|b|c!" + + code, errors = compiler(PRINT_FUNCTION_WITH_NONE_SEPARATOR)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == "a b\n" + + code, errors = compiler(PRINT_FUNCTION_WITH_NONE_END)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == "a b\n" + + code, errors = compiler(PRINT_FUNCTION_WITH_NONE_FILE)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == "a b\n" + + +ALLOWED_PRINT_FUNCTION_WITH_STAR_ARGS = """ +from __future__ import print_function +to_print = (1, 2, 3) +print(*to_print) +""" + + +def test_print_function_with_star_args(mocker): + _apply_ = mocker.stub() + _apply_.side_effect = lambda func, *args, **kwargs: func(*args, **kwargs) + + glb = { + '_print_': PrintCollector, + '_getattr_': None, + "_apply_": _apply_ + } + + code, errors = compiler(ALLOWED_PRINT_FUNCTION_WITH_STAR_ARGS)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == "1 2 3\n" + _apply_.assert_called_once_with(glb['_print']._call_print, 1, 2, 3) + + +ALLOWED_PRINT_FUNCTION_WITH_KWARGS = """ +from __future__ import print_function +to_print = (1, 2, 3) +kwargs = {'sep': '-', 'end': '!', 'file': None} +print(*to_print, **kwargs) +""" + + +def test_print_function_with_kw_args(mocker): + _apply_ = mocker.stub() + _apply_.side_effect = lambda func, *args, **kwargs: func(*args, **kwargs) + + glb = { + '_print_': PrintCollector, + '_getattr_': None, + "_apply_": _apply_ + } + + code, errors = compiler(ALLOWED_PRINT_FUNCTION_WITH_KWARGS)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == "1-2-3!" + _apply_.assert_called_once_with( + glb['_print']._call_print, + 1, + 2, + 3, + end='!', + file=None, + sep='-') + + +PROTECT_WRITE_ON_FILE = """ +from __future__ import print_function +print ('a', 'b', file=stream) +""" + + +def test_print_function__protect_file(mocker): + _getattr_ = mocker.stub() + _getattr_.side_effect = getattr + stream = mocker.stub() + stream.write = mocker.stub() + + glb = { + '_print_': PrintCollector, + '_getattr_': _getattr_, + 'stream': stream + } + + code, errors = compiler(PROTECT_WRITE_ON_FILE)[:2] + assert code is not None + assert errors == () + + exec(code, glb) + + _getattr_.assert_called_once_with(stream, 'write') + stream.write.assert_has_calls([ + mocker.call('a'), + mocker.call(' '), + mocker.call('b'), + mocker.call('\n') + ]) + + +# 'printed' is scope aware. +# => on a new function scope a new printed is generated. +INJECT_PRINT_COLLECTOR_NESTED = """ +from __future__ import print_function +def f2(): + return 'f2' + +def f1(): + print ('f1') + + def inner(): + print ('inner') + return printed + + return inner() + printed + f2() + +def main(): + print ('main') + return f1() + printed +""" + + +def test_print_function__nested_print_collector(): + code, errors = compiler(INJECT_PRINT_COLLECTOR_NESTED)[:2] + + glb = {"_print_": PrintCollector, '_getattr_': None} + exec(code, glb) + + ret = glb['main']() + assert ret == 'inner\nf1\nf2main\n' + + +WARN_PRINTED_NO_PRINT = """ +def foo(): + return printed +""" + + +def test_print_function__with_printed_no_print(): + code, errors, warnings = compiler(WARN_PRINTED_NO_PRINT)[:3] + + assert code is not None + assert errors == () + assert warnings == ["Line 2: Doesn't print, but reads 'printed' variable."] + + +WARN_PRINTED_NO_PRINT_NESTED = """ +from __future__ import print_function +print ('a') +def foo(): + return printed +printed +""" + + +def test_print_function__with_printed_no_print_nested(): + code, errors, warnings = compiler(WARN_PRINTED_NO_PRINT_NESTED)[:3] + + assert code is not None + assert errors == () + assert warnings == ["Line 4: Doesn't print, but reads 'printed' variable."] + + +WARN_PRINT_NO_PRINTED = """ +from __future__ import print_function +def foo(): + print (1) +""" + + +def test_print_function__with_print_no_printed(): + code, errors, warnings = compiler(WARN_PRINT_NO_PRINTED)[:3] + + assert code is not None + assert errors == () + assert warnings == ["Line 3: Prints, but never reads 'printed' variable."] + + +WARN_PRINT_NO_PRINTED_NESTED = """ +from __future__ import print_function +print ('a') +def foo(): + print ('x') +printed +""" + + +def test_print_function__with_print_no_printed_nested(): + code, errors, warnings = compiler(WARN_PRINT_NO_PRINTED_NESTED)[:3] + + assert code is not None + assert errors == () + assert warnings == ["Line 4: Prints, but never reads 'printed' variable."] + + +# python generates a new frame/scope for: +# modules, functions, class, lambda, all the comprehensions +# For class, lambda and comprehensions *no* new print collector scope should be +# generated. + +NO_PRINT_SCOPES = """ +from __future__ import print_function +def class_scope(): + class A: + print ('a') + return printed + +def lambda_scope(): + func = lambda x: print(x) + func(1) + func(2) + return printed + +def comprehension_scope(): + [print(1) for _ in range(2)] + return printed +""" + + +def test_print_function_no_new_scope(): + code, errors = compiler(NO_PRINT_SCOPES)[:2] + glb = { + '_print_': PrintCollector, + '__metaclass__': type, + '_getattr_': None, + '_getiter_': lambda ob: ob + } + exec(code, glb) + + ret = glb['class_scope']() + assert ret == 'a\n' + + ret = glb['lambda_scope']() + assert ret == '1\n2\n' + + ret = glb['comprehension_scope']() + assert ret == '1\n1\n' + + +PASS_PRINT_FUNCTION = """ +from __future__ import print_function +def main(): + def do_stuff(func): + func(1) + func(2) + + do_stuff(print) + return printed +""" + + +def test_print_function_pass_print_function(): + code, errors = compiler(PASS_PRINT_FUNCTION)[:2] + glb = {'_print_': PrintCollector, '_getattr_': None} + exec(code, glb) + + ret = glb['main']() + assert ret == '1\n2\n' + + +CONDITIONAL_PRINT = """ +from __future__ import print_function +def func(cond): + if cond: + print(1) + return printed +""" + + +def test_print_function_conditional_print(): + code, errors = compiler(CONDITIONAL_PRINT)[:2] + glb = {'_print_': PrintCollector, '_getattr_': None} + exec(code, glb) + + assert glb['func'](True) == '1\n' + assert glb['func'](False) == '' diff --git a/tests/test_print_stmt.py b/tests/test_print_stmt.py new file mode 100644 index 0000000..1165600 --- /dev/null +++ b/tests/test_print_stmt.py @@ -0,0 +1,280 @@ +from RestrictedPython._compat import IS_PY3 +from RestrictedPython.PrintCollector import PrintCollector +from tests import c_exec + +import pytest +import RestrictedPython + + +pytestmark = pytest.mark.skipif( + IS_PY3, + reason="print statement no longer exists in Python 3") + + +ALLOWED_PRINT_STATEMENT = """ +print 'Hello World!' +""" + +ALLOWED_PRINT_STATEMENT_WITH_NO_NL = """ +print 'Hello World!', +""" + +ALLOWED_MULTI_PRINT_STATEMENT = """ +print 'Hello World!', 'Hello Earth!' +""" + +# It looks like a function, but is still a statement in python2.X +ALLOWED_PRINT_TUPLE = """ +print('Hello World!') +""" + + +ALLOWED_PRINT_MULTI_TUPLE = """ +print('Hello World!', 'Hello Earth!') +""" + + +@pytest.mark.parametrize(*c_exec) +def test_print_stmt__simple_prints(c_exec): + glb = {'_print_': PrintCollector, '_getattr_': None} + + code, errors = c_exec(ALLOWED_PRINT_STATEMENT)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == 'Hello World!\n' + + code, errors = c_exec(ALLOWED_PRINT_STATEMENT_WITH_NO_NL)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == 'Hello World!' + + code, errors = c_exec(ALLOWED_MULTI_PRINT_STATEMENT)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == 'Hello World! Hello Earth!\n' + + code, errors = c_exec(ALLOWED_PRINT_TUPLE)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == "Hello World!\n" + + code, errors = c_exec(ALLOWED_PRINT_MULTI_TUPLE)[:2] + assert code is not None + assert errors == () + exec(code, glb) + assert glb['_print']() == "('Hello World!', 'Hello Earth!')\n" + + +@pytest.mark.parametrize(*c_exec) +def test_print_stmt__fail_with_none_target(c_exec, mocker): + code, errors = c_exec('print >> None, "test"')[:2] + + assert code is not None + assert errors == () + + glb = {'_getattr_': getattr, '_print_': PrintCollector} + + with pytest.raises(AttributeError) as excinfo: + exec(code, glb) + + assert "'NoneType' object has no attribute 'write'" in str(excinfo.value) + + +PROTECT_PRINT_STATEMENT_WITH_CHEVRON = """ +def print_into_stream(stream): + print >> stream, 'Hello World!' +""" + + +@pytest.mark.parametrize(*c_exec) +def test_print_stmt__protect_chevron_print(c_exec, mocker): + code, errors = c_exec(PROTECT_PRINT_STATEMENT_WITH_CHEVRON)[:2] + + _getattr_ = mocker.stub() + _getattr_.side_effect = getattr + glb = {'_getattr_': _getattr_, '_print_': PrintCollector} + + exec(code, glb) + + stream = mocker.stub() + stream.write = mocker.stub() + glb['print_into_stream'](stream) + + stream.write.assert_has_calls([ + mocker.call('Hello World!'), + mocker.call('\n') + ]) + + _getattr_.assert_called_once_with(stream, 'write') + + +# 'printed' is scope aware. +# => on a new function scope a new printed is generated. +INJECT_PRINT_COLLECTOR_NESTED = """ +def f2(): + return 'f2' + +def f1(): + print 'f1' + + def inner(): + print 'inner' + return printed + + return inner() + printed + f2() + +def main(): + print 'main' + return f1() + printed +""" + + +@pytest.mark.parametrize(*c_exec) +def test_print_stmt__nested_print_collector(c_exec, mocker): + code, errors = c_exec(INJECT_PRINT_COLLECTOR_NESTED)[:2] + + glb = {"_print_": PrintCollector, '_getattr_': None} + exec(code, glb) + + ret = glb['main']() + assert ret == 'inner\nf1\nf2main\n' + + +WARN_PRINTED_NO_PRINT = """ +def foo(): + return printed +""" + + +@pytest.mark.parametrize(*c_exec) +def test_print_stmt__with_printed_no_print(c_exec): + code, errors, warnings = c_exec(WARN_PRINTED_NO_PRINT)[:3] + + assert code is not None + assert errors == () + + if c_exec is RestrictedPython.compile.compile_restricted_exec: + assert warnings == [ + "Line 2: Doesn't print, but reads 'printed' variable."] + + if c_exec is RestrictedPython.RCompile.compile_restricted_exec: + assert warnings == ["Doesn't print, but reads 'printed' variable."] + + +WARN_PRINTED_NO_PRINT_NESTED = """ +print 'a' +def foo(): + return printed +printed +""" + + +@pytest.mark.parametrize(*c_exec) +def test_print_stmt__with_printed_no_print_nested(c_exec): + code, errors, warnings = c_exec(WARN_PRINTED_NO_PRINT_NESTED)[:3] + + assert code is not None + assert errors == () + + if c_exec is RestrictedPython.compile.compile_restricted_exec: + assert warnings == [ + "Line 2: Print statement is deprecated and not avaliable anymore in Python 3.", # NOQA: E501 + "Line 3: Doesn't print, but reads 'printed' variable." + ] + + if c_exec is RestrictedPython.RCompile.compile_restricted_exec: + assert warnings == ["Doesn't print, but reads 'printed' variable."] + + +WARN_PRINT_NO_PRINTED = """ +def foo(): + print 1 +""" + + +@pytest.mark.parametrize(*c_exec) +def test_print_stmt__with_print_no_printed(c_exec): + code, errors, warnings = c_exec(WARN_PRINT_NO_PRINTED)[:3] + + assert code is not None + assert errors == () + + if c_exec is RestrictedPython.compile.compile_restricted_exec: + assert warnings == [ + "Line 3: Print statement is deprecated and not avaliable anymore in Python 3.", # NOQA: E501 + "Line 2: Prints, but never reads 'printed' variable." + ] + + if c_exec is RestrictedPython.RCompile.compile_restricted_exec: + assert warnings == ["Prints, but never reads 'printed' variable."] + + +WARN_PRINT_NO_PRINTED_NESTED = """ +print 'a' +def foo(): + print 'x' +printed +""" + + +@pytest.mark.parametrize(*c_exec) +def test_print_stmt__with_print_no_printed_nested(c_exec): + code, errors, warnings = c_exec(WARN_PRINT_NO_PRINTED_NESTED)[:3] + + assert code is not None + assert errors == () + + if c_exec is RestrictedPython.compile.compile_restricted_exec: + assert warnings == [ + "Line 2: Print statement is deprecated and not avaliable anymore in Python 3.", # NOQA: E501 + "Line 4: Print statement is deprecated and not avaliable anymore in Python 3.", # NOQA: E501 + "Line 3: Prints, but never reads 'printed' variable.", + ] + + if c_exec is RestrictedPython.RCompile.compile_restricted_exec: + assert warnings == ["Prints, but never reads 'printed' variable."] + + +# python2 generates a new frame/scope for: +# modules, functions, class, lambda +# Since print statement cannot be used in lambda only ensure that no new scope +# for classes is generated. + +NO_PRINT_SCOPES = """ +def class_scope(): + class A: + print 'a' + return printed +""" + + +@pytest.mark.parametrize(*c_exec) +def test_print_stmt_no_new_scope(c_exec): + code, errors = c_exec(NO_PRINT_SCOPES)[:2] + glb = {'_print_': PrintCollector, '_getattr_': None} + exec(code, glb) + + ret = glb['class_scope']() + assert ret == 'a\n' + + +CONDITIONAL_PRINT = """ +def func(cond): + if cond: + print 1 + return printed +""" + + +@pytest.mark.parametrize(*c_exec) +def test_print_stmt_conditional_print(c_exec): + code, errors = c_exec(CONDITIONAL_PRINT)[:2] + glb = {'_print_': PrintCollector, '_getattr_': None} + exec(code, glb) + + assert glb['func'](True) == '1\n' + assert glb['func'](False) == '' diff --git a/tests/transformer/operators/test_arithmetic_operators.py b/tests/transformer/operators/test_arithmetic_operators.py new file mode 100644 index 0000000..9861c72 --- /dev/null +++ b/tests/transformer/operators/test_arithmetic_operators.py @@ -0,0 +1,55 @@ +from RestrictedPython._compat import IS_PY35_OR_GREATER +from tests import c_eval +from tests import e_eval + +import pytest + + +# Arithmetic Operators + + +@pytest.mark.parametrize(*e_eval) +def test_Add(e_eval): + assert e_eval('1 + 1') == 2 + + +@pytest.mark.parametrize(*e_eval) +def test_Sub(e_eval): + assert e_eval('5 - 3') == 2 + + +@pytest.mark.parametrize(*e_eval) +def test_Mult(e_eval): + assert e_eval('2 * 2') == 4 + + +@pytest.mark.parametrize(*e_eval) +def test_Div(e_eval): + assert e_eval('10 / 2') == 5 + + +@pytest.mark.parametrize(*e_eval) +def test_Mod(e_eval): + assert e_eval('10 % 3') == 1 + + +@pytest.mark.parametrize(*e_eval) +def test_Pow(e_eval): + assert e_eval('2 ** 8') == 256 + + +@pytest.mark.parametrize(*e_eval) +def test_FloorDiv(e_eval): + assert e_eval('7 // 2') == 3 + + +@pytest.mark.skipif( + not IS_PY35_OR_GREATER, + reason="MatMult was introducted on Python 3.5") +@pytest.mark.parametrize(*c_eval) +def test_MatMult(c_eval): + result = c_eval('(8, 3, 5) @ (2, 7, 1)') + assert result.errors == ( + 'Line None: MatMult statements are not allowed.', + ) + assert result.code is None diff --git a/tests/transformer/operators/test_bit_wise_operators.py b/tests/transformer/operators/test_bit_wise_operators.py new file mode 100644 index 0000000..7bbb001 --- /dev/null +++ b/tests/transformer/operators/test_bit_wise_operators.py @@ -0,0 +1,33 @@ +from tests import e_eval + +import pytest + + +@pytest.mark.parametrize(*e_eval) +def test_BitAnd(e_eval): + assert e_eval('5 & 3') == 1 + + +@pytest.mark.parametrize(*e_eval) +def test_BitOr(e_eval): + assert e_eval('5 | 3') == 7 + + +@pytest.mark.parametrize(*e_eval) +def test_BitXor(e_eval): + assert e_eval('5 ^ 3') == 6 + + +@pytest.mark.parametrize(*e_eval) +def test_Invert(e_eval): + assert e_eval('~17') == -18 + + +@pytest.mark.parametrize(*e_eval) +def test_LShift(e_eval): + assert e_eval('8 << 2') == 32 + + +@pytest.mark.parametrize(*e_eval) +def test_RShift(e_eval): + assert e_eval('8 >> 1') == 4 diff --git a/tests/transformer/operators/test_bool_operators.py b/tests/transformer/operators/test_bool_operators.py new file mode 100644 index 0000000..f31c6fe --- /dev/null +++ b/tests/transformer/operators/test_bool_operators.py @@ -0,0 +1,18 @@ +from tests import e_eval + +import pytest + + +@pytest.mark.parametrize(*e_eval) +def test_Or(e_eval): + assert e_eval('False or True') is True + + +@pytest.mark.parametrize(*e_eval) +def test_And(e_eval): + assert e_eval('True and True') is True + + +@pytest.mark.parametrize(*e_eval) +def test_Not(e_eval): + assert e_eval('not False') is True diff --git a/tests/transformer/operators/test_comparison_operators.py b/tests/transformer/operators/test_comparison_operators.py new file mode 100644 index 0000000..9d6545d --- /dev/null +++ b/tests/transformer/operators/test_comparison_operators.py @@ -0,0 +1,33 @@ +from tests import e_eval + +import pytest + + +@pytest.mark.parametrize(*e_eval) +def test_Eq(e_eval): + assert e_eval('1 == 1') is True + + +@pytest.mark.parametrize(*e_eval) +def test_NotEq(e_eval): + assert e_eval('1 != 2') is True + + +@pytest.mark.parametrize(*e_eval) +def test_Gt(e_eval): + assert e_eval('2 > 1') is True + + +@pytest.mark.parametrize(*e_eval) +def test_Lt(e_eval): + assert e_eval('1 < 2') + + +@pytest.mark.parametrize(*e_eval) +def test_GtE(e_eval): + assert e_eval('2 >= 2') is True + + +@pytest.mark.parametrize(*e_eval) +def test_LtE(e_eval): + assert e_eval('1 <= 2') is True diff --git a/tests/transformer/operators/test_identity_operators.py b/tests/transformer/operators/test_identity_operators.py new file mode 100644 index 0000000..526362e --- /dev/null +++ b/tests/transformer/operators/test_identity_operators.py @@ -0,0 +1,13 @@ +from tests import e_eval + +import pytest + + +@pytest.mark.parametrize(*e_eval) +def test_Is(e_eval): + assert e_eval('True is True') is True + + +@pytest.mark.parametrize(*e_eval) +def test_NotIs(e_eval): + assert e_eval('1 is not True') is True diff --git a/tests/transformer/operators/test_logical_operators.py b/tests/transformer/operators/test_logical_operators.py new file mode 100644 index 0000000..1d572c8 --- /dev/null +++ b/tests/transformer/operators/test_logical_operators.py @@ -0,0 +1,13 @@ +from tests import e_eval + +import pytest + + +@pytest.mark.parametrize(*e_eval) +def test_In(e_eval): + assert e_eval('1 in [1, 2, 3]') is True + + +@pytest.mark.parametrize(*e_eval) +def test_NotIn(e_eval): + assert e_eval('4 not in [1, 2, 3]') is True diff --git a/tests/transformer/operators/test_unary_operators.py b/tests/transformer/operators/test_unary_operators.py new file mode 100644 index 0000000..abab659 --- /dev/null +++ b/tests/transformer/operators/test_unary_operators.py @@ -0,0 +1,13 @@ +from tests import e_eval + +import pytest + + +@pytest.mark.parametrize(*e_eval) +def test_UAdd(e_eval): + assert e_eval('+a', {'a': 42}) == 42 + + +@pytest.mark.parametrize(*e_eval) +def test_USub(e_eval): + assert e_eval('-a', {'a': 2411}) == -2411 diff --git a/tests/transformer/test_async.py b/tests/transformer/test_async.py new file mode 100644 index 0000000..f09a738 --- /dev/null +++ b/tests/transformer/test_async.py @@ -0,0 +1,109 @@ +from RestrictedPython import compile_restricted_exec +from RestrictedPython._compat import IS_PY35_OR_GREATER +from RestrictedPython.transformer import RestrictingNodeTransformer +from tests import c_exec + +import pytest + + +pytestmark = pytest.mark.skipif( + not IS_PY35_OR_GREATER, + reason="async statement was first introduced in Python 3.5") + + +# Example from https://docs.python.org/3/library/asyncio-task.html +ASYNC_DEF_EXMAPLE = """ +import asyncio + +async def hello_world(): + print() + +loop = asyncio.get_event_loop() +# Blocking call which returns when the hello_world() coroutine is done +loop.run_until_complete(hello_world()) +loop.close() +""" + + +@pytest.mark.parametrize(*c_exec) +def test_async_def(c_exec): + result = c_exec(ASYNC_DEF_EXMAPLE) + assert result.errors == ( + 'Line 4: AsyncFunctionDef statements are not allowed.', + ) + assert result.code is None + + +class RestrictingAsyncNodeTransformer(RestrictingNodeTransformer): + """Transformer which allows `async def` for the tests.""" + + def visit_AsyncFunctionDef(self, node): + """Allow `async def`. + + This is needed to get the function body to be parsed thus allowing + to catch `await`, `async for` and `async with`. + """ + return self.node_contents_visit(node) + + +# Modified example from https://docs.python.org/3/library/asyncio-task.html +AWAIT_EXAMPLE = """ +import asyncio +import datetime + +async def display_date(loop): + end_time = loop.time() + 5.0 + while True: + print(datetime.datetime.now()) + if (loop.time() + 1.0) >= end_time: + break + await asyncio.sleep(1) + +loop = asyncio.get_event_loop() +# Blocking call which returns when the display_date() coroutine is done +loop.run_until_complete(display_date(loop)) +loop.close() +""" + + +@pytest.mark.parametrize(*c_exec) +def test_await(c_exec): + result = compile_restricted_exec( + AWAIT_EXAMPLE, + policy=RestrictingAsyncNodeTransformer) + assert result.errors == ('Line 11: Await statements are not allowed.',) + assert result.code is None + + +# Modified example https://www.python.org/dev/peps/pep-0525/ +ASYNC_WITH_EXAMPLE = """ +async def square_series(con, to): + async with con.transaction(): + print(con) +""" + + +@pytest.mark.parametrize(*c_exec) +def test_async_with(c_exec): + result = compile_restricted_exec( + ASYNC_WITH_EXAMPLE, + policy=RestrictingAsyncNodeTransformer) + assert result.errors == ('Line 3: AsyncWith statements are not allowed.',) + assert result.code is None + + +# Modified example https://www.python.org/dev/peps/pep-0525/ +ASYNC_FOR_EXAMPLE = """ +async def read_rows(rows): + async for row in rows: + yield row +""" + + +@pytest.mark.parametrize(*c_exec) +def test_async_for(c_exec): + result = compile_restricted_exec( + ASYNC_FOR_EXAMPLE, + policy=RestrictingAsyncNodeTransformer) + assert result.errors == ('Line 3: AsyncFor statements are not allowed.',) + assert result.code is None diff --git a/tests/transformer/test_base_types.py b/tests/transformer/test_base_types.py new file mode 100644 index 0000000..9dc0867 --- /dev/null +++ b/tests/transformer/test_base_types.py @@ -0,0 +1,32 @@ +from RestrictedPython._compat import IS_PY2 +from tests import c_exec +from tests import e_eval + +import pytest + + +@pytest.mark.parametrize(*e_eval) +def test_Num(e_eval): + """It allows to use number literals.""" + assert e_eval('42') == 42 + + +@pytest.mark.parametrize(*e_eval) +def test_Bytes(e_eval): + """It allows to use bytes literals.""" + assert e_eval('b"code"') == b"code" + + +@pytest.mark.parametrize(*e_eval) +def test_Set(e_eval): + """It allows to use set literals.""" + assert e_eval('{1, 2, 3}') == set([1, 2, 3]) + + +@pytest.mark.skipif(IS_PY2, + reason="... is new in Python 3") +@pytest.mark.parametrize(*c_exec) +def test_Ellipsis(c_exec): + """It prevents using the `ellipsis` statement.""" + result = c_exec('...') + assert result.errors == ('Line 1: Ellipsis statements are not allowed.',) diff --git a/tests/transformer/test_global_local.py b/tests/transformer/test_global_local.py new file mode 100644 index 0000000..34ea1ee --- /dev/null +++ b/tests/transformer/test_global_local.py @@ -0,0 +1,45 @@ +from RestrictedPython._compat import IS_PY3 +from tests import c_exec +from tests import e_exec + +import pytest + + +GLOBAL_EXAMPLE = """ +def x(): + global a + a = 11 +x() +""" + + +@pytest.mark.parametrize(*e_exec) +def test_Global(e_exec): + glb = {'a': None} + e_exec(GLOBAL_EXAMPLE, glb) + assert glb['a'] == 11 + + +# Example from: +# https://www.smallsurething.com/a-quick-guide-to-nonlocal-in-python-3/ +NONLOCAL_EXAMPLE = """ +def outside(): + msg = "Outside!" + def inside(): + nonlocal msg + msg = "Inside!" + print(msg) + inside() + print(msg) +outside() +""" + + +@pytest.mark.skipif( + not IS_PY3, + reason="The `nonlocal` statement was introduced in Python 3.0.") +@pytest.mark.parametrize(*c_exec) +def test_Nonlocal(c_exec): + result = c_exec(NONLOCAL_EXAMPLE) + assert result.errors == ('Line 5: Nonlocal statements are not allowed.',) + assert result.code is None diff --git a/tests/transformer/test_slice.py b/tests/transformer/test_slice.py new file mode 100644 index 0000000..6908aa9 --- /dev/null +++ b/tests/transformer/test_slice.py @@ -0,0 +1,23 @@ +from operator import getitem +from tests import e_eval + +import pytest + + +@pytest.mark.parametrize(*e_eval) +def test_slice(e_eval): + low = 1 + high = 4 + stride = 3 + + rglb = {'_getitem_': getitem} # restricted globals + + assert e_eval('[1, 2, 3, 4, 5]', rglb) == [1, 2, 3, 4, 5] + assert e_eval('[1, 2, 3, 4, 5][:]', rglb) == [1, 2, 3, 4, 5] + assert e_eval('[1, 2, 3, 4, 5][%d:]' % low, rglb) == [2, 3, 4, 5] + assert e_eval('[1, 2, 3, 4, 5][:%d]' % high, rglb) == [1, 2, 3, 4] + assert e_eval('[1, 2, 3, 4, 5][%d:%d]' % (low, high), rglb) == [2, 3, 4] + assert e_eval('[1, 2, 3, 4, 5][::%d]' % stride, rglb) == [1, 4] + assert e_eval('[1, 2, 3, 4, 5][%d::%d]' % (low, stride), rglb) == [2, 5] + assert e_eval('[1, 2, 3, 4, 5][:%d:%d]' % (high, stride), rglb) == [1, 4] + assert e_eval('[1, 2, 3, 4, 5][%d:%d:%d]' % (low, high, stride), rglb) == [2] # NOQA: E501 diff --git a/tests/transformer/test_subscript.py b/tests/transformer/test_subscript.py new file mode 100644 index 0000000..6a74fc1 --- /dev/null +++ b/tests/transformer/test_subscript.py @@ -0,0 +1,172 @@ +from tests import e_exec + +import pytest + + +SIMPLE_SUBSCRIPTS = """ +def simple_subscript(a): + return a['b'] +""" + + +@pytest.mark.parametrize(*e_exec) +def test_read_simple_subscript(e_exec, mocker): + value = None + _getitem_ = mocker.stub() + _getitem_.side_effect = lambda ob, index: (ob, index) + glb = {'_getitem_': _getitem_} + e_exec(SIMPLE_SUBSCRIPTS, glb) + + assert (value, 'b') == glb['simple_subscript'](value) + + +TUPLE_SUBSCRIPTS = """ +def tuple_subscript(a): + return a[1, 2] +""" + + +@pytest.mark.parametrize(*e_exec) +def test_tuple_subscript(e_exec, mocker): + value = None + _getitem_ = mocker.stub() + _getitem_.side_effect = lambda ob, index: (ob, index) + glb = {'_getitem_': _getitem_} + e_exec(TUPLE_SUBSCRIPTS, glb) + + assert (value, (1, 2)) == glb['tuple_subscript'](value) + + +SLICE_SUBSCRIPT_NO_UPPER_BOUND = """ +def slice_subscript_no_upper_bound(a): + return a[1:] +""" + + +@pytest.mark.parametrize(*e_exec) +def test_read_slice_subscript_no_upper_bound(e_exec, mocker): + value = None + _getitem_ = mocker.stub() + _getitem_.side_effect = lambda ob, index: (ob, index) + glb = {'_getitem_': _getitem_} + e_exec(SLICE_SUBSCRIPT_NO_UPPER_BOUND, glb) + + assert (value, slice(1, None, None)) == glb['slice_subscript_no_upper_bound'](value) # NOQA: E501 + + +SLICE_SUBSCRIPT_NO_LOWER_BOUND = """ +def slice_subscript_no_lower_bound(a): + return a[:1] +""" + + +@pytest.mark.parametrize(*e_exec) +def test_read_slice_subscript_no_lower_bound(e_exec, mocker): + value = None + _getitem_ = mocker.stub() + _getitem_.side_effect = lambda ob, index: (ob, index) + glb = {'_getitem_': _getitem_} + e_exec(SLICE_SUBSCRIPT_NO_LOWER_BOUND, glb) + + assert (value, slice(None, 1, None)) == glb['slice_subscript_no_lower_bound'](value) # NOQA: E501 + + +SLICE_SUBSCRIPT_NO_STEP = """ +def slice_subscript_no_step(a): + return a[1:2] +""" + + +@pytest.mark.parametrize(*e_exec) +def test_read_slice_subscript_no_step(e_exec, mocker): + value = None + _getitem_ = mocker.stub() + _getitem_.side_effect = lambda ob, index: (ob, index) + glb = {'_getitem_': _getitem_} + e_exec(SLICE_SUBSCRIPT_NO_STEP, glb) + + assert (value, slice(1, 2, None)) == glb['slice_subscript_no_step'](value) + + +SLICE_SUBSCRIPT_WITH_STEP = """ +def slice_subscript_with_step(a): + return a[1:2:3] +""" + + +@pytest.mark.parametrize(*e_exec) +def test_read_slice_subscript_with_step(e_exec, mocker): + value = None + _getitem_ = mocker.stub() + _getitem_.side_effect = lambda ob, index: (ob, index) + glb = {'_getitem_': _getitem_} + e_exec(SLICE_SUBSCRIPT_WITH_STEP, glb) + + assert (value, slice(1, 2, 3)) == glb['slice_subscript_with_step'](value) + + +EXTENDED_SLICE_SUBSCRIPT = """ + +def extended_slice_subscript(a): + return a[0, :1, 1:, 1:2, 1:2:3] +""" + + +@pytest.mark.parametrize(*e_exec) +def test_read_extended_slice_subscript(e_exec, mocker): + value = None + _getitem_ = mocker.stub() + _getitem_.side_effect = lambda ob, index: (ob, index) + glb = {'_getitem_': _getitem_} + e_exec(EXTENDED_SLICE_SUBSCRIPT, glb) + ret = glb['extended_slice_subscript'](value) + ref = ( + value, + ( + 0, + slice(None, 1, None), + slice(1, None, None), + slice(1, 2, None), + slice(1, 2, 3) + ) + ) + + assert ref == ret + + +WRITE_SUBSCRIPTS = """ +def assign_subscript(a): + a['b'] = 1 +""" + + +@pytest.mark.parametrize(*e_exec) +def test_write_subscripts( + e_exec, mocker): + value = {'b': None} + _write_ = mocker.stub() + _write_.side_effect = lambda ob: ob + glb = {'_write_': _write_} + e_exec(WRITE_SUBSCRIPTS, glb) + + glb['assign_subscript'](value) + assert value['b'] == 1 + + +DEL_SUBSCRIPT = """ +def del_subscript(a): + del a['b'] +""" + + +@pytest.mark.parametrize(*e_exec) +def test_del_subscripts( + e_exec, mocker): + value = {'b': None} + _write_ = mocker.stub() + _write_.side_effect = lambda ob: ob + glb = {'_write_': _write_} + e_exec(DEL_SUBSCRIPT, glb) + glb['del_subscript'](value) + + assert value == {} diff --git a/tests/transformer/test_transformer.py b/tests/transformer/test_transformer.py new file mode 100644 index 0000000..0e2933c --- /dev/null +++ b/tests/transformer/test_transformer.py @@ -0,0 +1,1758 @@ +from RestrictedPython import RestrictingNodeTransformer +from RestrictedPython._compat import IS_PY2 +from RestrictedPython._compat import IS_PY3 +from RestrictedPython.Guards import guarded_iter_unpack_sequence +from RestrictedPython.Guards import guarded_unpack_sequence +from RestrictedPython.Guards import safe_builtins +from tests import c_exec +from tests import e_eval +from tests import e_exec + +import ast +import contextlib +import pytest +import RestrictedPython +import types + + +def test_transformer__RestrictingNodeTransformer__generic_visit__1(): + """It log an error if there is an unknown ast node visited.""" + class MyFancyNode(ast.AST): + pass + + transformer = RestrictingNodeTransformer() + transformer.visit(MyFancyNode()) + assert transformer.errors == [ + 'Line None: MyFancyNode statements are not allowed.'] + assert transformer.warnings == [ + 'Line None: MyFancyNode statement is not known to RestrictedPython'] + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Call__1(c_exec): + """It compiles a function call successfully and returns the used name.""" + result = c_exec('a = max([1, 2, 3])') + assert result.errors == () + loc = {} + exec(result.code, {}, loc) + assert loc['a'] == 3 + assert result.used_names == {'max': True} + + +EXEC_STATEMENT = """\ +def no_exec(): + exec 'q = 1' +""" + + +@pytest.mark.skipif(IS_PY3, + reason="exec statement no longer exists in Python 3") +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Exec__1(c_exec): + """It prevents using the `exec` statement. (Python 2 only)""" + result = c_exec(EXEC_STATEMENT) + assert result.errors == ('Line 2: Exec statements are not allowed.',) + + +BAD_NAME_STARTING_WITH_UNDERSCORE = """\ +def bad_name(): + __ = 12 +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__1(c_exec): + """It denies a variable name starting in `__`.""" + result = c_exec(BAD_NAME_STARTING_WITH_UNDERSCORE) + assert result.errors == ( + 'Line 2: "__" is an invalid variable name because it starts with "_"',) + + +BAD_NAME_OVERRIDE_GUARD_WITH_NAME = """\ +def overrideGuardWithName(): + _getattr = None +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__2(c_exec): + """It denies a variable name starting in `_`.""" + result = c_exec(BAD_NAME_OVERRIDE_GUARD_WITH_NAME) + assert result.errors == ( + 'Line 2: "_getattr" is an invalid variable name because ' + 'it starts with "_"',) + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__2_5(e_exec): + """It allows `_` as variable name.""" + glb = e_exec('_ = 2411') + assert glb['_'] == 2411 + + +BAD_NAME_OVERRIDE_OVERRIDE_GUARD_WITH_FUNCTION = """\ +def overrideGuardWithFunction(): + def _getattr(o): + return o +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__3(c_exec): + """It denies a function name starting in `_`.""" + result = c_exec(BAD_NAME_OVERRIDE_OVERRIDE_GUARD_WITH_FUNCTION) + assert result.errors == ( + 'Line 2: "_getattr" is an invalid variable name because it ' + 'starts with "_"',) + + +BAD_NAME_OVERRIDE_GUARD_WITH_CLASS = """\ +def overrideGuardWithClass(): + class _getattr: + pass +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__4(c_exec): + """It denies a class name starting in `_`.""" + result = c_exec(BAD_NAME_OVERRIDE_GUARD_WITH_CLASS) + assert result.errors == ( + 'Line 2: "_getattr" is an invalid variable name because it ' + 'starts with "_"',) + + +BAD_NAME_IN_WITH = """\ +def with_as_bad_name(): + with x as _leading_underscore: + pass +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__4_4(c_exec): + """It denies a variable name in with starting in `_`.""" + result = c_exec(BAD_NAME_IN_WITH) + assert result.errors == ( + 'Line 2: "_leading_underscore" is an invalid variable name because ' + 'it starts with "_"',) + + +BAD_NAME_IN_COMPOUND_WITH = """\ +def compound_with_bad_name(): + with a as b, c as _restricted_name: + pass +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__4_5(c_exec): + """It denies a variable name in with starting in `_`.""" + result = c_exec(BAD_NAME_IN_COMPOUND_WITH) + assert result.errors == ( + 'Line 2: "_restricted_name" is an invalid variable name because ' + 'it starts with "_"',) + + +BAD_NAME_DICT_COMP = """\ +def dict_comp_bad_name(): + {y: y for _restricted_name in x} +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__4_6(c_exec): + """It denies a variable name starting in `_` in a dict comprehension.""" + result = c_exec(BAD_NAME_DICT_COMP) + assert result.errors == ( + 'Line 2: "_restricted_name" is an invalid variable name because ' + 'it starts with "_"',) + + +BAD_NAME_SET_COMP = """\ +def set_comp_bad_name(): + {y for _restricted_name in x} +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__4_7(c_exec): + """It denies a variable name starting in `_` in a dict comprehension.""" + result = c_exec(BAD_NAME_SET_COMP) + assert result.errors == ( + 'Line 2: "_restricted_name" is an invalid variable name because ' + 'it starts with "_"',) + + +BAD_NAME_ENDING_WITH___ROLES__ = """\ +def bad_name(): + myvar__roles__ = 12 +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__5(c_exec): + """It denies a variable name ending in `__roles__`.""" + result = c_exec(BAD_NAME_ENDING_WITH___ROLES__) + assert result.errors == ( + 'Line 2: "myvar__roles__" is an invalid variable name because it ' + 'ends with "__roles__".',) + + +BAD_NAME_PRINTED = """\ +def bad_name(): + printed = 12 +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__6(c_exec): + """It denies a variable named `printed`.""" + result = c_exec(BAD_NAME_PRINTED) + assert result.errors == ('Line 2: "printed" is a reserved name.',) + + +BAD_NAME_PRINT = """\ +def bad_name(): + def print(): + pass +""" + + +@pytest.mark.skipif(IS_PY2, + reason="print is a statement in Python 2") +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Name__7(c_exec): + """It denies a variable named `print`.""" + result = c_exec(BAD_NAME_PRINT) + assert result.errors == ('Line 2: "print" is a reserved name.',) + + +BAD_ATTR_UNDERSCORE = """\ +def bad_attr(): + some_ob = object() + some_ob._some_attr = 15 +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Attribute__1(c_exec): + """It is an error if a bad attribute name is used.""" + result = c_exec(BAD_ATTR_UNDERSCORE) + assert result.errors == ( + 'Line 3: "_some_attr" is an invalid attribute name because it ' + 'starts with "_".',) + + +BAD_ATTR_ROLES = """\ +def bad_attr(): + some_ob = object() + some_ob.abc__roles__ +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Attribute__2(c_exec): + """It is an error if a bad attribute name is used.""" + result = c_exec(BAD_ATTR_ROLES) + assert result.errors == ( + 'Line 3: "abc__roles__" is an invalid attribute name because it ' + 'ends with "__roles__".',) + + +TRANSFORM_ATTRIBUTE_ACCESS = """\ +def func(): + return a.b +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Attribute__3( + e_exec, mocker): + """It transforms the attribute access to `_getattr_`.""" + glb = { + '_getattr_': mocker.stub(), + 'a': [], + 'b': 'b' + } + e_exec(TRANSFORM_ATTRIBUTE_ACCESS, glb) + glb['func']() + glb['_getattr_'].assert_called_once_with([], 'b') + + +ALLOW_UNDERSCORE_ONLY = """\ +def func(): + some_ob = object() + some_ob._ +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Attribute__4(c_exec): + """It allows `_` as attribute name.""" + result = c_exec(ALLOW_UNDERSCORE_ONLY) + assert result.errors == () + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Attribute__5( + e_exec, mocker): + """It transforms writing to an attribute to `_write_`.""" + glb = { + '_write_': mocker.stub(), + 'a': mocker.stub(), + } + glb['_write_'].return_value = glb['a'] + + e_exec("a.b = 'it works'", glb) + + glb['_write_'].assert_called_once_with(glb['a']) + assert glb['a'].b == 'it works' + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Attribute__5_5( + e_exec, mocker): + """It transforms deleting of an attribute to `_write_`.""" + glb = { + '_write_': mocker.stub(), + 'a': mocker.stub(), + } + glb['a'].b = 'it exists' + glb['_write_'].return_value = glb['a'] + + e_exec("del a.b", glb) + + glb['_write_'].assert_called_once_with(glb['a']) + assert not hasattr(glb['a'], 'b') + + +DISALLOW_TRACEBACK_ACCESS = """ +try: + raise Exception() +except Exception as e: + tb = e.__traceback__ +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Attribute__6(c_exec): + """It denies access to the __traceback__ attribute.""" + result = c_exec(DISALLOW_TRACEBACK_ACCESS) + assert result.errors == ( + 'Line 5: "__traceback__" is an invalid attribute name because ' + 'it starts with "_".',) + + +TRANSFORM_ATTRIBUTE_ACCESS_FUNCTION_DEFAULT = """ +def func_default(x=a.a): + return x +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Attribute__7( + e_exec, mocker): + """It transforms attribute access in function default kw to `_write_`.""" + _getattr_ = mocker.Mock() + _getattr_.side_effect = getattr + + glb = { + '_getattr_': _getattr_, + 'a': mocker.Mock(a=1), + } + + e_exec(TRANSFORM_ATTRIBUTE_ACCESS_FUNCTION_DEFAULT, glb) + + _getattr_.assert_has_calls([mocker.call(glb['a'], 'a')]) + assert glb['func_default']() == 1 + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Attribute__8( + e_exec, mocker): + """It transforms attribute access in lamda default kw to `_write_`.""" + _getattr_ = mocker.Mock() + _getattr_.side_effect = getattr + + glb = { + '_getattr_': _getattr_, + 'b': mocker.Mock(b=2) + } + + e_exec('lambda_default = lambda x=b.b: x', glb) + + _getattr_.assert_has_calls([mocker.call(glb['b'], 'b')]) + assert glb['lambda_default']() == 2 + + +EXEC_FUNCTION = """\ +def no_exec(): + exec('q = 1') +""" + + +@pytest.mark.skipif(IS_PY2, + reason="exec is a statement in Python 2") +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Call__2(c_exec): + """It is an error if the code call the `exec` function.""" + result = c_exec(EXEC_FUNCTION) + assert result.errors == ("Line 2: Exec calls are not allowed.",) + + +EVAL_FUNCTION = """\ +def no_eval(): + eval('q = 1') +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Call__3(c_exec): + """It is an error if the code call the `eval` function.""" + result = c_exec(EVAL_FUNCTION) + if c_exec is RestrictedPython.compile.compile_restricted_exec: + assert result.errors == ("Line 2: Eval calls are not allowed.",) + else: + # `eval()` is allowed in the old implementation. :-( + assert result.errors == () + + +ITERATORS = """ +def for_loop(it): + c = 0 + for a in it: + c = c + a + return c + + +def nested_for_loop(it1, it2): + c = 0 + for a in it1: + for b in it2: + c = c + a + b + return c + +def dict_comp(it): + return {a: a + a for a in it} + +def list_comp(it): + return [a + a for a in it] + +def nested_list_comp(it1, it2): + return [a + b for a in it1 if a > 1 for b in it2] + +def set_comp(it): + return {a + a for a in it} + +def generator(it): + return (a + a for a in it) + +def nested_generator(it1, it2): + return (a+b for a in it1 if a > 0 for b in it2) +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__guard_iter(e_exec, mocker): + it = (1, 2, 3) + _getiter_ = mocker.stub() + _getiter_.side_effect = lambda x: x + glb = {'_getiter_': _getiter_} + e_exec(ITERATORS, glb) + + ret = glb['for_loop'](it) + assert 6 == ret + _getiter_.assert_called_once_with(it) + _getiter_.reset_mock() + + ret = glb['nested_for_loop']((1, 2), (3, 4)) + assert 20 == ret + _getiter_.assert_has_calls([ + mocker.call((1, 2)), + mocker.call((3, 4)) + ]) + _getiter_.reset_mock() + + ret = glb['dict_comp'](it) + assert {1: 2, 2: 4, 3: 6} == ret + _getiter_.assert_called_once_with(it) + _getiter_.reset_mock() + + ret = glb['list_comp'](it) + assert [2, 4, 6] == ret + _getiter_.assert_called_once_with(it) + _getiter_.reset_mock() + + ret = glb['nested_list_comp']((1, 2), (3, 4)) + assert [5, 6] == ret + _getiter_.assert_has_calls([ + mocker.call((1, 2)), + mocker.call((3, 4)) + ]) + _getiter_.reset_mock() + + ret = glb['set_comp'](it) + assert {2, 4, 6} == ret + _getiter_.assert_called_once_with(it) + _getiter_.reset_mock() + + ret = glb['generator'](it) + assert isinstance(ret, types.GeneratorType) + assert list(ret) == [2, 4, 6] + _getiter_.assert_called_once_with(it) + _getiter_.reset_mock() + + ret = glb['nested_generator']((0, 1, 2), (1, 2)) + assert isinstance(ret, types.GeneratorType) + assert list(ret) == [2, 3, 3, 4] + _getiter_.assert_has_calls([ + mocker.call((0, 1, 2)), + mocker.call((1, 2)), + mocker.call((1, 2))]) + _getiter_.reset_mock() + + +ITERATORS_WITH_UNPACK_SEQUENCE = """ +def for_loop(it): + c = 0 + for (a, b) in it: + c = c + a + b + return c + +def dict_comp(it): + return {a: a + b for (a, b) in it} + +def list_comp(it): + return [a + b for (a, b) in it] + +def set_comp(it): + return {a + b for (a, b) in it} + +def generator(it): + return (a + b for (a, b) in it) +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__guard_iter2(e_exec, mocker): + it = ((1, 2), (3, 4), (5, 6)) + + call_ref = [ + mocker.call(it), + mocker.call(it[0]), + mocker.call(it[1]), + mocker.call(it[2]) + ] + + _getiter_ = mocker.stub() + _getiter_.side_effect = lambda x: x + + glb = { + '_getiter_': _getiter_, + '_iter_unpack_sequence_': guarded_iter_unpack_sequence + } + + e_exec(ITERATORS_WITH_UNPACK_SEQUENCE, glb) + + ret = glb['for_loop'](it) + assert ret == 21 + _getiter_.assert_has_calls(call_ref) + _getiter_.reset_mock() + + ret = glb['dict_comp'](it) + assert ret == {1: 3, 3: 7, 5: 11} + _getiter_.assert_has_calls(call_ref) + _getiter_.reset_mock() + + ret = glb['list_comp'](it) + assert ret == [3, 7, 11] + _getiter_.assert_has_calls(call_ref) + _getiter_.reset_mock() + + ret = glb['set_comp'](it) + assert ret == {3, 7, 11} + _getiter_.assert_has_calls(call_ref) + _getiter_.reset_mock() + + # The old code did not run with unpack sequence inside generators + if compile == RestrictedPython.compile.compile_restricted_exec: + ret = list(glb['generator'](it)) + assert ret == [3, 7, 11] + _getiter_.assert_has_calls(call_ref) + _getiter_.reset_mock() + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_AugAssign__1( + e_exec, mocker): + """It allows augmented assign for variables.""" + _inplacevar_ = mocker.stub() + _inplacevar_.side_effect = lambda op, val, expr: val + expr + + glb = { + '_inplacevar_': _inplacevar_, + 'a': 1, + 'x': 1, + 'z': 0 + } + + e_exec("a += x + z", glb) + assert glb['a'] == 2 + _inplacevar_.assert_called_once_with('+=', 1, 1) + _inplacevar_.reset_mock() + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_AugAssign__2(c_exec): + """It forbids augmented assign of attributes.""" + result = c_exec("a.a += 1") + assert result.errors == ( + 'Line 1: Augmented assignment of attributes is not allowed.',) + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_AugAssign__3(c_exec): + """It forbids augmented assign of subscripts.""" + result = c_exec("a[a] += 1") + assert result.errors == ( + 'Line 1: Augmented assignment of object items and slices is not ' + 'allowed.',) + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_AugAssign__4(c_exec): + """It forbids augmented assign of slices.""" + result = c_exec("a[x:y] += 1") + assert result.errors == ( + 'Line 1: Augmented assignment of object items and slices is not ' + 'allowed.',) + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_AugAssign__5(c_exec): + """It forbids augmented assign of slices with steps.""" + result = c_exec("a[x:y:z] += 1") + assert result.errors == ( + 'Line 1: Augmented assignment of object items and slices is not ' + 'allowed.',) + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Assert__1(e_exec): + """It allows assert statements.""" + e_exec('assert 1') + + +# def f(a, b, c): pass +# f(*two_element_sequence, **dict_with_key_c) +# +# makes the elements of two_element_sequence +# visible to f via its 'a' and 'b' arguments, +# and the dict_with_key_c['c'] value visible via its 'c' argument. +# It is a devious way to extract values without going through security checks. + +FUNCTIONC_CALLS = """ +star = (3, 4) +kwargs = {'x': 5, 'y': 6} + +def positional_args(): + return foo(1, 2) + +def star_args(): + return foo(*star) + +def positional_and_star_args(): + return foo(1, 2, *star) + +def kw_args(): + return foo(**kwargs) + +def star_and_kw(): + return foo(*star, **kwargs) + +def positional_and_star_and_kw_args(): + return foo(1, *star, **kwargs) + +def positional_and_star_and_keyword_and_kw_args(): + return foo(1, 2, *star, r=9, **kwargs) +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Call(e_exec, mocker): + _apply_ = mocker.stub() + _apply_.side_effect = lambda func, *args, **kwargs: func(*args, **kwargs) + + glb = { + '_apply_': _apply_, + 'foo': lambda *args, **kwargs: (args, kwargs) + } + + e_exec(FUNCTIONC_CALLS, glb) + + ret = glb['positional_args']() + assert ((1, 2), {}) == ret + assert _apply_.called is False + _apply_.reset_mock() + + ret = glb['star_args']() + ref = ((3, 4), {}) + assert ref == ret + _apply_.assert_called_once_with(glb['foo'], *ref[0]) + _apply_.reset_mock() + + ret = glb['positional_and_star_args']() + ref = ((1, 2, 3, 4), {}) + assert ref == ret + _apply_.assert_called_once_with(glb['foo'], *ref[0]) + _apply_.reset_mock() + + ret = glb['kw_args']() + ref = ((), {'x': 5, 'y': 6}) + assert ref == ret + _apply_.assert_called_once_with(glb['foo'], **ref[1]) + _apply_.reset_mock() + + ret = glb['star_and_kw']() + ref = ((3, 4), {'x': 5, 'y': 6}) + assert ref == ret + _apply_.assert_called_once_with(glb['foo'], *ref[0], **ref[1]) + _apply_.reset_mock() + + ret = glb['positional_and_star_and_kw_args']() + ref = ((1, 3, 4), {'x': 5, 'y': 6}) + assert ref == ret + _apply_.assert_called_once_with(glb['foo'], *ref[0], **ref[1]) + _apply_.reset_mock() + + ret = glb['positional_and_star_and_keyword_and_kw_args']() + ref = ((1, 2, 3, 4), {'x': 5, 'y': 6, 'r': 9}) + assert ref == ret + _apply_.assert_called_once_with(glb['foo'], *ref[0], **ref[1]) + _apply_.reset_mock() + + +functiondef_err_msg = 'Line 1: "_bad" is an invalid variable ' \ + 'name because it starts with "_"' + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_FunctionDef__1( + c_exec): + """It prevents function arguments starting with `_`.""" + result = c_exec("def foo(_bad): pass") + # RestrictedPython.compile.compile_restricted_exec on Python 2 renders + # the error message twice. This is necessary as otherwise *_bad and **_bad + # would be allowed. + assert functiondef_err_msg in result.errors + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_FunctionDef__2( + c_exec): + """It prevents function keyword arguments starting with `_`.""" + result = c_exec("def foo(_bad=1): pass") + # RestrictedPython.compile.compile_restricted_exec on Python 2 renders + # the error message twice. This is necessary as otherwise *_bad and **_bad + # would be allowed. + assert functiondef_err_msg in result.errors + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_FunctionDef__3( + c_exec): + """It prevents function * arguments starting with `_`.""" + result = c_exec("def foo(*_bad): pass") + assert result.errors == (functiondef_err_msg,) + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_FunctionDef__4( + c_exec): + """It prevents function ** arguments starting with `_`.""" + result = c_exec("def foo(**_bad): pass") + assert result.errors == (functiondef_err_msg,) + + +@pytest.mark.skipif( + IS_PY3, + reason="tuple parameter unpacking is gone in Python 3") +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_FunctionDef__5( + c_exec): + """It prevents function arguments starting with `_` in tuples.""" + result = c_exec("def foo((a, _bad)): pass") + # RestrictedPython.compile.compile_restricted_exec on Python 2 renders + # the error message twice. This is necessary as otherwise *_bad and **_bad + # would be allowed. + assert functiondef_err_msg in result.errors + + +@pytest.mark.skipif( + IS_PY3, + reason="tuple parameter unpacking is gone in Python 3") +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_FunctionDef__6( + c_exec): + """It prevents function arguments starting with `_` in tuples.""" + # The old `compile` breaks with tuples in function arguments: + if c_exec is RestrictedPython.compile.compile_restricted_exec: + result = c_exec("def foo(a, (c, (_bad, c))): pass") + # RestrictedPython.compile.compile_restricted_exec on Python 2 renders + # the error message twice. This is necessary as otherwise *_bad and + # **_bad would be allowed. + assert functiondef_err_msg in result.errors + + +@pytest.mark.skipif( + IS_PY2, + reason="There is no single `*` argument in Python 2") +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_FunctionDef__7( + c_exec): + """It prevents `_` function arguments together with a single `*`.""" + result = c_exec("def foo(good, *, _bad): pass") + assert result.errors == (functiondef_err_msg,) + + +NESTED_SEQ_UNPACK = """ +def nested((a, b, (c, (d, e)))): + return a, b, c, d, e + +def nested_with_order((a, b), (c, d)): + return a, b, c, d +""" + + +@pytest.mark.skipif( + IS_PY3, + reason="tuple parameter unpacking is gone in python 3") +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_FunctionDef_2( + e_exec, mocker): + _getiter_ = mocker.stub() + _getiter_.side_effect = lambda it: it + + glb = { + '_getiter_': _getiter_, + '_unpack_sequence_': guarded_unpack_sequence + } + + e_exec('def simple((a, b)): return a, b', glb) + + val = (1, 2) + ret = glb['simple'](val) + assert ret == val + _getiter_.assert_called_once_with(val) + _getiter_.reset_mock() + + try: + e_exec(NESTED_SEQ_UNPACK, glb) + except AttributeError: + # The old RCompile did not support nested. + return + + val = (1, 2, (3, (4, 5))) + ret = glb['nested'](val) + assert ret == (1, 2, 3, 4, 5) + assert 3 == _getiter_.call_count + _getiter_.assert_any_call(val) + _getiter_.assert_any_call(val[2]) + _getiter_.assert_any_call(val[2][1]) + _getiter_.reset_mock() + + ret = glb['nested_with_order']((1, 2), (3, 4)) + assert ret == (1, 2, 3, 4) + _getiter_.assert_has_calls([ + mocker.call((1, 2)), + mocker.call((3, 4))]) + _getiter_.reset_mock() + + +lambda_err_msg = 'Line 1: "_bad" is an invalid variable ' \ + 'name because it starts with "_"' + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Lambda__1(c_exec): + """It prevents arguments starting with `_`.""" + result = c_exec("lambda _bad: None") + # RestrictedPython.compile.compile_restricted_exec on Python 2 renders + # the error message twice. This is necessary as otherwise *_bad and **_bad + # would be allowed. + assert lambda_err_msg in result.errors + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Lambda__2(c_exec): + """It prevents keyword arguments starting with `_`.""" + result = c_exec("lambda _bad=1: None") + # RestrictedPython.compile.compile_restricted_exec on Python 2 renders + # the error message twice. This is necessary as otherwise *_bad and **_bad + # would be allowed. + assert lambda_err_msg in result.errors + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Lambda__3(c_exec): + """It prevents * arguments starting with `_`.""" + result = c_exec("lambda *_bad: None") + assert result.errors == (lambda_err_msg,) + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Lambda__4(c_exec): + """It prevents ** arguments starting with `_`.""" + result = c_exec("lambda **_bad: None") + assert result.errors == (lambda_err_msg,) + + +@pytest.mark.skipif( + IS_PY3, + reason="tuple parameter unpacking is gone in Python 3") +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Lambda__5(c_exec): + """It prevents arguments starting with `_` in tuple unpacking.""" + # The old `compile` breaks with tuples in arguments: + if c_exec is RestrictedPython.compile.compile_restricted_exec: + result = c_exec("lambda (a, _bad): None") + # RestrictedPython.compile.compile_restricted_exec on Python 2 renders + # the error message twice. This is necessary as otherwise *_bad and + # **_bad would be allowed. + assert lambda_err_msg in result.errors + + +@pytest.mark.skipif( + IS_PY3, + reason="tuple parameter unpacking is gone in Python 3") +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Lambda__6(c_exec): + """It prevents arguments starting with `_` in nested tuple unpacking.""" + # The old `compile` breaks with tuples in arguments: + if c_exec is RestrictedPython.compile.compile_restricted_exec: + result = c_exec("lambda (a, (c, (_bad, c))): None") + # RestrictedPython.compile.compile_restricted_exec on Python 2 renders + # the error message twice. This is necessary as otherwise *_bad and + # **_bad would be allowed. + assert lambda_err_msg in result.errors + + +@pytest.mark.skipif( + IS_PY2, + reason="There is no single `*` argument in Python 2") +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Lambda__7(c_exec): + """It prevents arguments starting with `_` together with a single `*`.""" + result = c_exec("lambda good, *, _bad: None") + assert result.errors == (lambda_err_msg,) + + +BAD_ARG_IN_LAMBDA = """\ +def check_getattr_in_lambda(arg=lambda _bad=(lambda ob, name: name): _bad2): + 42 +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Lambda__8(c_exec): + """It prevents arguments starting with `_` in weird lambdas.""" + result = c_exec(BAD_ARG_IN_LAMBDA) + # RestrictedPython.compile.compile_restricted_exec finds both invalid + # names, while the old implementation seems to abort after the first. + assert lambda_err_msg in result.errors + + +@pytest.mark.skipif( + IS_PY3, + reason="tuple parameter unpacking is gone in python 3") +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Lambda_2( + e_exec, mocker): + _getiter_ = mocker.stub() + _getiter_.side_effect = lambda it: it + glb = { + '_getiter_': _getiter_, + '_unpack_sequence_': guarded_unpack_sequence, + '_getattr_': lambda ob, val: getattr(ob, val) + } + + src = "m = lambda (a, (b, c)), *ag, **kw: a+b+c+sum(ag)+sum(kw.values())" + try: + e_exec(src, glb) + except AttributeError: + # Old implementation does not support tuple unpacking + return + + ret = glb['m']((1, (2, 3)), 4, 5, 6, g=7, e=8) + assert ret == 36 + assert 2 == _getiter_.call_count + _getiter_.assert_any_call((1, (2, 3))) + _getiter_.assert_any_call((2, 3)) + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Assign(e_exec, mocker): + src = "orig = (a, (x, z)) = (c, d) = g" + + _getiter_ = mocker.stub() + _getiter_.side_effect = lambda it: it + + glb = { + '_getiter_': _getiter_, + '_unpack_sequence_': guarded_unpack_sequence, + 'g': (1, (2, 3)), + } + + e_exec(src, glb) + assert glb['a'] == 1 + assert glb['x'] == 2 + assert glb['z'] == 3 + assert glb['c'] == 1 + assert glb['d'] == (2, 3) + assert glb['orig'] == (1, (2, 3)) + assert _getiter_.call_count == 3 + _getiter_.assert_any_call((1, (2, 3))) + _getiter_.assert_any_call((2, 3)) + _getiter_.reset_mock() + + +@pytest.mark.skipif( + IS_PY2, + reason="starred assignments are python3 only") +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Assign2( + e_exec, mocker): + src = "a, *d, (c, *e), x = (1, 2, 3, (4, 3, 4), 5)" + + _getiter_ = mocker.stub() + _getiter_.side_effect = lambda it: it + + glb = { + '_getiter_': _getiter_, + '_unpack_sequence_': guarded_unpack_sequence + } + + e_exec(src, glb) + assert glb['a'] == 1 + assert glb['d'] == [2, 3] + assert glb['c'] == 4 + assert glb['e'] == [3, 4] + assert glb['x'] == 5 + + _getiter_.assert_has_calls([ + mocker.call((1, 2, 3, (4, 3, 4), 5)), + mocker.call((4, 3, 4))]) + + +TRY_EXCEPT = """ +def try_except(m): + try: + m('try') + raise IndentationError('f1') + except IndentationError as error: + m('except') +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Try__1( + e_exec, mocker): + """It allows try-except statements.""" + trace = mocker.stub() + e_exec(TRY_EXCEPT)['try_except'](trace) + + trace.assert_has_calls([ + mocker.call('try'), + mocker.call('except') + ]) + + +TRY_EXCEPT_ELSE = """ +def try_except_else(m): + try: + m('try') + except: + m('except') + else: + m('else') +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Try__2( + e_exec, mocker): + """It allows try-except-else statements.""" + trace = mocker.stub() + e_exec(TRY_EXCEPT_ELSE)['try_except_else'](trace) + + trace.assert_has_calls([ + mocker.call('try'), + mocker.call('else') + ]) + + +TRY_FINALLY = """ +def try_finally(m): + try: + m('try') + 1 / 0 + finally: + m('finally') + return +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_TryFinally__1( + e_exec, mocker): + """It allows try-finally statements.""" + trace = mocker.stub() + e_exec(TRY_FINALLY)['try_finally'](trace) + + trace.assert_has_calls([ + mocker.call('try'), + mocker.call('finally') + ]) + + +TRY_EXCEPT_FINALLY = """ +def try_except_finally(m): + try: + m('try') + 1 / 0 + except: + m('except') + finally: + m('finally') +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_TryFinally__2( + e_exec, mocker): + """It allows try-except-finally statements.""" + trace = mocker.stub() + e_exec(TRY_EXCEPT_FINALLY)['try_except_finally'](trace) + + trace.assert_has_calls([ + mocker.call('try'), + mocker.call('except'), + mocker.call('finally') + ]) + + +TRY_EXCEPT_ELSE_FINALLY = """ +def try_except_else_finally(m): + try: + m('try') + except: + m('except') + else: + m('else') + finally: + m('finally') +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_TryFinally__3( + e_exec, mocker): + """It allows try-except-else-finally statements.""" + trace = mocker.stub() + e_exec(TRY_EXCEPT_ELSE_FINALLY)['try_except_else_finally'](trace) + + trace.assert_has_calls([ + mocker.call('try'), + mocker.call('else'), + mocker.call('finally') + ]) + + +EXCEPT_WITH_TUPLE_UNPACK = """ +def tuple_unpack(err): + try: + raise err + except Exception as (a, (b, c)): + return a + b + c +""" + + +@pytest.mark.skipif( + IS_PY3, + reason="tuple unpacking on exceptions is gone in python3") +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_ExceptHandler( + e_exec, mocker): + _getiter_ = mocker.stub() + _getiter_.side_effect = lambda it: it + + glb = { + '_getiter_': _getiter_, + '_unpack_sequence_': guarded_unpack_sequence + } + + e_exec(EXCEPT_WITH_TUPLE_UNPACK, glb) + err = Exception(1, (2, 3)) + ret = glb['tuple_unpack'](err) + assert ret == 6 + + _getiter_.assert_has_calls([ + mocker.call(err), + mocker.call((2, 3))]) + + +BAD_TRY_EXCEPT = """ +def except_using_bad_name(): + try: + foo + except NameError as _leading_underscore: + # The name of choice (say, _write) is now assigned to an exception + # object. Hard to exploit, but conceivable. + pass +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_ExceptHandler__2( + c_exec): + """It denies bad names in the except as statement.""" + result = c_exec(BAD_TRY_EXCEPT) + assert result.errors == ( + 'Line 5: "_leading_underscore" is an invalid variable name because ' + 'it starts with "_"',) + + +import_errmsg = ( + 'Line 1: "%s" is an invalid variable name because it starts with "_"') + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Import__1(c_exec): + """It allows importing a module.""" + result = c_exec('import a') + assert result.errors == () + assert result.code is not None + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Import__2(c_exec): + """It denies importing a module starting with `_`.""" + result = c_exec('import _a') + assert result.errors == (import_errmsg % '_a',) + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Import__3(c_exec): + """It denies importing a module starting with `_` as something.""" + result = c_exec('import _a as m') + assert result.errors == (import_errmsg % '_a',) + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Import__4(c_exec): + """It denies importing a module as something starting with `_`.""" + result = c_exec('import a as _m') + assert result.errors == (import_errmsg % '_m',) + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Import__5(c_exec): + """It allows importing from a module.""" + result = c_exec('from a import m') + assert result.errors == () + assert result.code is not None + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Import_6(c_exec): + """It allows importing from a module starting with `_`.""" + result = c_exec('from _a import m') + assert result.errors == () + assert result.code is not None + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Import__7(c_exec): + """It denies importing from a module as something starting with `_`.""" + result = c_exec('from a import m as _n') + assert result.errors == (import_errmsg % '_n',) + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Import__8(c_exec): + """It denies as-importing something starting with `_` from a module.""" + result = c_exec('from a import _m as n') + assert result.errors == (import_errmsg % '_m',) + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_Import__9(c_exec): + """It denies relative from importing as something starting with `_`.""" + result = c_exec('from .x import y as _leading_underscore') + assert result.errors == (import_errmsg % '_leading_underscore',) + + +GOOD_CLASS = ''' +class Good: + pass +''' + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_ClassDef__1(c_exec): + """It allows to define an class.""" + result = c_exec(GOOD_CLASS) + assert result.errors == () + assert result.code is not None + + +BAD_CLASS = '''\ +class _bad: + pass +''' + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_ClassDef__2(c_exec): + """It does not allow class names which start with an underscore.""" + result = c_exec(BAD_CLASS) + assert result.errors == ( + 'Line 1: "_bad" is an invalid variable name ' + 'because it starts with "_"',) + + +IMPLICIT_METACLASS = ''' +class Meta: + pass + +b = Meta().foo +''' + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_ClassDef__3(e_exec): + """It applies the global __metaclass__ to all generated classes if present. + """ + def _metaclass(name, bases, dict): + ob = type(name, bases, dict) + ob.foo = 2411 + return ob + + restricted_globals = dict( + __metaclass__=_metaclass, b=None, _getattr_=getattr) + + e_exec(IMPLICIT_METACLASS, restricted_globals) + + assert restricted_globals['b'] == 2411 + + +EXPLICIT_METACLASS = ''' +class WithMeta(metaclass=MyMetaClass): + pass +''' + + +@pytest.mark.skipif(IS_PY2, reason="No valid syntax in Python 2.") +@pytest.mark.parametrize(*c_exec) +def test_transformer__RestrictingNodeTransformer__visit_ClassDef__4(c_exec): + """It does not allow to pass a metaclass to class definitions.""" + + result = c_exec(EXPLICIT_METACLASS) + + assert result.errors == ( + 'Line 2: The keyword argument "metaclass" is not allowed.',) + assert result.code is None + + +DECORATED_CLASS = '''\ +def wrap(cls): + cls.wrap_att = 23 + return cls + +class Base: + base_att = 42 + +@wrap +class Combined(Base): + class_att = 2342 + +comb = Combined() +''' + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_ClassDef__5(e_exec): + """It preserves base classes and decorators for classes.""" + + restricted_globals = dict( + comb=None, _getattr_=getattr, _write_=lambda x: x, __metaclass__=type, + __name__='restricted_module', __builtins__=safe_builtins) + + e_exec(DECORATED_CLASS, restricted_globals) + + comb = restricted_globals['comb'] + assert comb.class_att == 2342 + assert comb.base_att == 42 + if e_exec.compile_func is RestrictedPython.compile.compile_restricted_exec: + # Class decorators are only supported by the new implementation. + assert comb.wrap_att == 23 + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__test_ternary_if( + e_exec, mocker): + src = 'x.y = y.a if y.z else y.b' + _getattr_ = mocker.stub() + _getattr_.side_effect = lambda ob, key: ob[key] + _write_ = mocker.stub() + _write_.side_effect = lambda ob: ob + + glb = { + '_getattr_': _getattr_, + '_write_': _write_, + 'x': mocker.stub(), + 'y': {'a': 'a', 'b': 'b'}, + } + + glb['y']['z'] = True + e_exec(src, glb) + + assert glb['x'].y == 'a' + _write_.assert_called_once_with(glb['x']) + _getattr_.assert_has_calls([ + mocker.call(glb['y'], 'z'), + mocker.call(glb['y'], 'a')]) + + _write_.reset_mock() + _getattr_.reset_mock() + + glb['y']['z'] = False + e_exec(src, glb) + + assert glb['x'].y == 'b' + _write_.assert_called_once_with(glb['x']) + _getattr_.assert_has_calls([ + mocker.call(glb['y'], 'z'), + mocker.call(glb['y'], 'b')]) + + +WHILE = """\ +a = 5 +while a < 7: + a = a + 3 +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_While__1(e_exec): + """It allows `while` statements.""" + glb = e_exec(WHILE) + assert glb['a'] == 8 + + +BREAK = """\ +a = 5 +while True: + a = a + 3 + if a >= 7: + break +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Break__1(e_exec): + """It allows `break` statements.""" + glb = e_exec(BREAK) + assert glb['a'] == 8 + + +CONTINUE = """\ +a = 3 +while a < 10: + if a < 5: + a = a + 1 + continue + a = a + 10 +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__RestrictingNodeTransformer__visit_Continue__1(e_exec): + """It allows `continue` statements.""" + glb = e_exec(CONTINUE) + assert glb['a'] == 15 + + +WITH_STMT_WITH_UNPACK_SEQUENCE = """ +def call(ctx): + with ctx() as (a, (c, b)): + return a, c, b +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer__with_stmt_unpack_sequence(e_exec, mocker): + @contextlib.contextmanager + def ctx(): + yield (1, (2, 3)) + + _getiter_ = mocker.stub() + _getiter_.side_effect = lambda ob: ob + + glb = { + '_getiter_': _getiter_, + '_unpack_sequence_': guarded_unpack_sequence + } + + e_exec(WITH_STMT_WITH_UNPACK_SEQUENCE, glb) + + ret = glb['call'](ctx) + + assert ret == (1, 2, 3) + _getiter_.assert_has_calls([ + mocker.call((1, (2, 3))), + mocker.call((2, 3))]) + + +WITH_STMT_MULTI_CTX_WITH_UNPACK_SEQUENCE = """ +def call(ctx1, ctx2): + with ctx1() as (a, (b, c)), ctx2() as ((x, z), (s, h)): + return a, b, c, x, z, s, h +""" + + +@pytest.mark.parametrize(*c_exec) +def test_transformer__with_stmt_multi_ctx_unpack_sequence(c_exec, mocker): + result = c_exec(WITH_STMT_MULTI_CTX_WITH_UNPACK_SEQUENCE) + assert result.errors == () + + @contextlib.contextmanager + def ctx1(): + yield (1, (2, 3)) + + @contextlib.contextmanager + def ctx2(): + yield (4, 5), (6, 7) + + _getiter_ = mocker.stub() + _getiter_.side_effect = lambda ob: ob + + glb = { + '_getiter_': _getiter_, + '_unpack_sequence_': guarded_unpack_sequence + } + + exec(result.code, glb) + + ret = glb['call'](ctx1, ctx2) + + assert ret == (1, 2, 3, 4, 5, 6, 7) + _getiter_.assert_has_calls([ + mocker.call((1, (2, 3))), + mocker.call((2, 3)), + mocker.call(((4, 5), (6, 7))), + mocker.call((4, 5)), + mocker.call((6, 7)) + ]) + + +WITH_STMT_ATTRIBUTE_ACCESS = """ +def simple(ctx): + with ctx as x: + x.z = x.y + 1 + +def assign_attr(ctx, x): + with ctx as x.y: + x.z = 1 + +def load_attr(w): + with w.ctx as x: + x.z = 1 + +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer_with_stmt_attribute_access(e_exec, mocker): + _getattr_ = mocker.stub() + _getattr_.side_effect = getattr + + _write_ = mocker.stub() + _write_.side_effect = lambda ob: ob + + glb = {'_getattr_': _getattr_, '_write_': _write_} + e_exec(WITH_STMT_ATTRIBUTE_ACCESS, glb) + + # Test simple + ctx = mocker.MagicMock(y=1) + ctx.__enter__.return_value = ctx + + glb['simple'](ctx) + + assert ctx.z == 2 + _write_.assert_called_once_with(ctx) + _getattr_.assert_called_once_with(ctx, 'y') + + _write_.reset_mock() + _getattr_.reset_mock() + + # Test assign_attr + x = mocker.Mock() + glb['assign_attr'](ctx, x) + + assert x.z == 1 + assert x.y == ctx + _write_.assert_has_calls([ + mocker.call(x), + mocker.call(x) + ]) + + _write_.reset_mock() + + # Test load_attr + ctx = mocker.MagicMock() + ctx.__enter__.return_value = ctx + + w = mocker.Mock(ctx=ctx) + + glb['load_attr'](w) + + assert w.ctx.z == 1 + _getattr_.assert_called_once_with(w, 'ctx') + _write_.assert_called_once_with(w.ctx) + + +WITH_STMT_SUBSCRIPT = """ +def single_key(ctx, x): + with ctx as x['key']: + pass + + +def slice_key(ctx, x): + with ctx as x[2:3]: + pass +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer_with_stmt_subscript(e_exec, mocker): + _write_ = mocker.stub() + _write_.side_effect = lambda ob: ob + + glb = {'_write_': _write_} + e_exec(WITH_STMT_SUBSCRIPT, glb) + + # Test single_key + ctx = mocker.MagicMock() + ctx.__enter__.return_value = ctx + x = {} + + glb['single_key'](ctx, x) + + assert x['key'] == ctx + _write_.assert_called_once_with(x) + _write_.reset_mock() + + # Test slice_key + ctx = mocker.MagicMock() + ctx.__enter__.return_value = (1, 2) + + x = [0, 0, 0, 0, 0, 0] + glb['slice_key'](ctx, x) + + assert x == [0, 0, 1, 2, 0, 0, 0] + _write_.assert_called_once_with(x) + + +DICT_COMPREHENSION_WITH_ATTRS = """ +def call(seq): + return {y.k: y.v for y in seq.z if y.k} +""" + + +@pytest.mark.parametrize(*e_exec) +def test_transformer_dict_comprehension_with_attrs(e_exec, mocker): + _getattr_ = mocker.Mock() + _getattr_.side_effect = getattr + + _getiter_ = mocker.Mock() + _getiter_.side_effect = lambda ob: ob + + glb = {'_getattr_': _getattr_, '_getiter_': _getiter_} + e_exec(DICT_COMPREHENSION_WITH_ATTRS, glb) + + z = [mocker.Mock(k=0, v='a'), mocker.Mock(k=1, v='b')] + seq = mocker.Mock(z=z) + + ret = glb['call'](seq) + assert ret == {1: 'b'} + + _getiter_.assert_called_once_with(z) + _getattr_.assert_has_calls([ + mocker.call(seq, 'z'), + mocker.call(z[0], 'k'), + mocker.call(z[1], 'k'), + mocker.call(z[1], 'v'), + mocker.call(z[1], 'k') + ]) + + +@pytest.mark.parametrize(*e_eval) +def test_transformer__RestrictingNodeTransformer__visit_Eq__1(e_eval): + """It allows == expressions.""" + assert e_eval('1 == int("1")') is True + + +@pytest.mark.parametrize(*e_eval) +def test_transformer__RestrictingNodeTransformer__visit_NotEq__1(e_eval): + """It allows != expressions.""" + assert e_eval('1 != int("1")') is False + + +@pytest.mark.parametrize(*e_eval) +def test_transformer__RestrictingNodeTransformer__visit_Lt__1(e_eval): + """It allows < expressions.""" + assert e_eval('1 < 3') is True + + +@pytest.mark.parametrize(*e_eval) +def test_transformer__RestrictingNodeTransformer__visit_LtE__1(e_eval): + """It allows < expressions.""" + assert e_eval('1 <= 3') is True + + +@pytest.mark.parametrize(*e_eval) +def test_transformer__RestrictingNodeTransformer__visit_Gt__1(e_eval): + """It allows > expressions.""" + assert e_eval('1 > 3') is False + + +@pytest.mark.parametrize(*e_eval) +def test_transformer__RestrictingNodeTransformer__visit_GtE__1(e_eval): + """It allows >= expressions.""" + assert e_eval('1 >= 3') is False + + +@pytest.mark.parametrize(*e_eval) +def test_transformer__RestrictingNodeTransformer__visit_Is__1(e_eval): + """It allows `is` expressions.""" + assert e_eval('None is None') is True + + +@pytest.mark.parametrize(*e_eval) +def test_transformer__RestrictingNodeTransformer__visit_IsNot__1(e_eval): + """It allows `is not` expressions.""" + assert e_eval('2 is not None') is True + + +@pytest.mark.parametrize(*e_eval) +def test_transformer__RestrictingNodeTransformer__visit_In__1(e_eval): + """It allows `in` expressions.""" + assert e_eval('2 in [1, 2, 3]') is True + + +@pytest.mark.parametrize(*e_eval) +def test_transformer__RestrictingNodeTransformer__visit_NotIn__1(e_eval): + """It allows `in` expressions.""" + assert e_eval('2 not in [1, 2, 3]') is False diff --git a/tests/transformer/test_yield.py b/tests/transformer/test_yield.py new file mode 100644 index 0000000..3a1ad03 --- /dev/null +++ b/tests/transformer/test_yield.py @@ -0,0 +1,38 @@ +from RestrictedPython._compat import IS_PY3 +from tests import c_exec + +import pytest + + +YIELD_EXAMPLE = """\ +def no_yield(): + yield 42 +""" + + +@pytest.mark.parametrize(*c_exec) +def test_yield(c_exec): + """It prevents using the `yield` statement.""" + result = c_exec(YIELD_EXAMPLE) + assert result.errors == ("Line 2: Yield statements are not allowed.",) + assert result.code is None + + +# Modified Example from http://stackabuse.com/python-async-await-tutorial/ +YIELD_FORM_EXAMPLE = """ +import asyncio + +@asyncio.coroutine +def get_json(client, url): + file_content = yield from load_file('data.ini') +""" + + +@pytest.mark.skipif( + not IS_PY3, + reason="`yield from` statement was first introduced in Python 3.3") +@pytest.mark.parametrize(*c_exec) +def test_yield_from(c_exec): + result = c_exec(YIELD_FORM_EXAMPLE) + assert result.errors == ('Line 6: YieldFrom statements are not allowed.',) + assert result.code is None diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..9e4c822 --- /dev/null +++ b/tox.ini @@ -0,0 +1,96 @@ +[tox] +envlist = + flake8, + coverage-clean, + py27, + py27-datetime, + py27-rp3, + py34, + py35, + py36, + py36-datetime, + pypy, + docs, + isort, + coverage-report, +skip_missing_interpreters = False + +[testenv] +usedevelop = True +extras = + develop + test +commands = + pytest --cov=src --cov-report=xml --html=report-{envname}.html --self-contained-html {posargs} + pytest --doctest-modules src/RestrictedPython/compile.py {posargs} +setenv = + COVERAGE_FILE=.coverage.{envname} +deps = + pytest + pytest-cov + pytest-remove-stale-bytecode + pytest-mock + pytest-html + +[testenv:py27-datetime] +basepython = python2.7 +deps = + {[testenv]deps} + DateTime + +[testenv:py36-datetime] +basepython = python3.6 +deps = + {[testenv]deps} + DateTime + +[testenv:py27-rp3] +basepython = python2.7 +commands = + coverage run {envbindir}/zope-testrunner --path=src/RestrictedPython --all {posargs} +deps = + .[test] + zope.testrunner + coverage + +[testenv:coverage-clean] +deps = coverage +skip_install = true +commands = coverage erase + +[testenv:coverage-report] +basepython = python2.7 +deps = coverage +setenv = + COVERAGE_FILE=.coverage +skip_install = true +commands = + coverage combine + coverage html + coverage xml + coverage report + +[testenv:isort] +basepython = python2.7 +deps = isort +commands = + isort --check-only --recursive {toxinidir}/src {toxinidir}/tests {posargs} + +[testenv:isort-apply] +basepython = python2.7 +deps = isort +commands = + isort --apply --recursive {toxinidir}/src {toxinidir}/tests {posargs} + +[testenv:flake8] +basepython = python2.7 +deps = flake8 +commands = flake8 --doctests src tests setup.py + +[testenv:docs] +basepython = python2.7 +commands = + sphinx-build -b html -d build/docs/doctrees docs build/docs/html + sphinx-build -b doctest docs build/docs/doctrees +deps = + .[docs]