Skip to content

Changes from Python upstream #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 26, 2019
Merged

Changes from Python upstream #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
22bb4da
Do not use system recursion limit from the time of import.
frenzymadness Nov 15, 2019
dca6f38
Do no allow unsupported combinations of ddir/stripdir/prependdir.
frenzymadness Nov 15, 2019
8fafc8f
Backport tests from CPython with windows compatibility.
frenzymadness Nov 15, 2019
9bed13c
Update readme
frenzymadness Nov 15, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 25 additions & 23 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,6 @@ ZeroDivisionError: division by zero

* ✓ Test it with Python packages in COPR

## ToDo

* Push it to Fedora rawhide

* ✓ %py_byte_compile RPM macro uses `compileall2` (done in [python-rpm-macros](https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/25))
Expand All @@ -96,17 +94,18 @@ ZeroDivisionError: division by zero
* ✓ Prepare patches for upstream CPython
* [Pull request](https://github.com/python/cpython/pull/16012) merged and will be part of Python 3.9

* ✓ Changes from upstream CPython backported back

## Testing

You can test it locally with tox or unittest directly:

```shell
$ python3 -m unittest test_compileall_original.py
............sss.......................sss.....................ss.................
----------------------------------------------------------------------
Ran 81 tests in 2.137s
$ python3 -m unittest test_compileall2.py
..............sss....ss.......................sss....ss.....................ss.............................----------------------------------------------------------------------
Ran 107 tests in 3.714s

OK (skipped=8)
OK (skipped=12)
```

but running in a Docker container might be better because the superuser has privileges to write to `sys.path` which lowers the number of skipped tests.
Expand All @@ -125,23 +124,26 @@ and run tests in it:

```shell
$ docker run --rm -it -e TOXENV=py37 -v $PWD:/src:Z -w /src compileall2
py37 create: /src/.tox/py37
py37 installdeps: pytest
py37 installed: atomicwrites==1.3.0,attrs==19.1.0,more-itertools==6.0.0,pluggy==0.9.0,py==1.8.0,pytest==4.3.1,six==1.12.0
py37 run-test-pre: PYTHONHASHSEED='519909491'
py37 runtests: commands[0] | pytest -v -s
========================================== test session starts ==========================================
platform linux -- Python 3.7.2, pytest-4.3.1, py-1.8.0, pluggy-0.9.0 -- /src/.tox/py37/bin/python
py37 installed: atomicwrites==1.3.0,attrs==19.3.0,compileall2==0.5.0,coverage==4.5.4,importlib-metadata==0.23,more-itertools==7.2.0,packaging==19.2,pluggy==0.13.0,py==1.8.0,pyparsing==2.4.5,pytest==5.2.3,six==1.13.0,wcwidth==0.1.7,zipp==0.6.0
py37 run-test-pre: PYTHONHASHSEED='1615314833'
py37 runtests: commands[0] | coverage run --append -m py.test
==================================== test session starts =====================================
platform linux -- Python 3.7.5, pytest-5.2.3, py-1.8.0, pluggy-0.13.0
cachedir: .tox/py37/.pytest_cache
rootdir: /src, inifile:
collected 81 items

test_compileall_original.py::CompileallTestsWithSourceEpoch::test_compile_dir_pathlike PASSED
test_compileall_original.py::CompileallTestsWithSourceEpoch::test_compile_file_pathlike PASSED
test_compileall_original.py::CompileallTestsWithSourceEpoch::test_compile_file_pathlike_ddir PASSED
... etc ...
================================= 79 passed, 2 skipped in 5.15 seconds ==================================
________________________________________________ summary ________________________________________________
rootdir: /src
collected 107 items
test_compileall2.py ............ss..................................................ss [ 61%]
..............................ss......... [100%]

=============================== 101 passed, 6 skipped in 7.40s ===============================
py37 runtests: commands[1] | coverage report -i '--omit=.tox/*'
Name Stmts Miss Cover
-----------------------------------------
compileall2.py 232 48 79%
test_compileall2.py 621 8 99%
-----------------------------------------
TOTAL 853 56 93%
__________________________________________ summary ___________________________________________
py37: commands succeeded
congratulations :)
```
Expand Down
30 changes: 20 additions & 10 deletions compileall2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,6 @@
pyc_header_lenght = 8
pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER)

RECURSION_LIMIT = sys.getrecursionlimit()

__all__ = ["compile_dir","compile_file","compile_path"]

def optimization_kwarg(opt):
Expand All @@ -60,7 +58,7 @@ def optimization_kwarg(opt):
else:
return dict()

def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):
def _walk_dir(dir, maxlevels, quiet=0):
if PY36 and quiet < 2 and isinstance(dir, os.PathLike):
dir = os.fspath(dir)
else:
Expand All @@ -85,7 +83,7 @@ def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):
yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
quiet=quiet)

def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,
def compile_dir(dir, maxlevels=None, ddir=None, force=False,
rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
invalidation_mode=None, stripdir=None,
prependdir=None, limit_sl_dest=None):
Expand Down Expand Up @@ -123,6 +121,8 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,
from concurrent.futures import ProcessPoolExecutor
except ImportError:
workers = 1
if maxlevels is None:
maxlevels = sys.getrecursionlimit()
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
success = True
if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
Expand Down Expand Up @@ -173,6 +173,11 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
limit_sl_dest: ignore symlinks if they are pointing outside of
the defined path.
"""

if ddir is not None and (stripdir is not None or prependdir is not None):
raise ValueError(("Destination dir (ddir) cannot be used "
"in combination with stripdir or prependdir"))

success = True
if PY36 and quiet < 2 and isinstance(fullname, os.PathLike):
fullname = os.fspath(fullname)
Expand Down Expand Up @@ -327,7 +332,7 @@ def main():
parser = argparse.ArgumentParser(
description='Utilities to support installing Python libraries.')
parser.add_argument('-l', action='store_const', const=0,
default=RECURSION_LIMIT, dest='maxlevels',
default=None, dest='maxlevels',
help="don't recurse into subdirectories")
parser.add_argument('-r', type=int, dest='recursion',
help=('control the maximum recursion level. '
Expand All @@ -349,16 +354,16 @@ def main():
default=None,
help=('part of path to left-strip from path '
'to source file - for example buildroot. '
'if `-d` and `-s` options are specified, '
'then `-d` takes precedence.'))
'`-d` and `-s` options cannot be '
'specified together.'))
parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir',
default=None,
help=('path to add as prefix to path '
'to source file - for example / to make '
'it absolute when some part is removed '
'by `-s` option'
'if `-d` and `-a` options are specified, '
'then `-d` takes precedence.'))
'by `-s` option. '
'`-d` and `-p` options cannot be '
'specified together.'))
parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
help=('skip files matching the regular expression; '
'the regexp is searched for in the full path '
Expand Down Expand Up @@ -408,6 +413,11 @@ def main():
if args.opt_levels is None:
args.opt_levels = [-1]

if args.ddir is not None and (
args.stripdir is not None or args.prependdir is not None
):
parser.error("-d cannot be used in combination with -s or -p")

# if flist is provided then load it
if args.flist:
try:
Expand Down
75 changes: 40 additions & 35 deletions test_compileall2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,16 +103,6 @@ def setUp(self):
os.mkdir(self.subdirectory)
self.source_path3 = os.path.join(self.subdirectory, '_test3.py')
shutil.copyfile(self.source_path, self.source_path3)
many_directories = [str(number) for number in range(1, 100)]
self.long_path = os.path.join(self.directory,
"long",
*many_directories)
os.makedirs(self.long_path)
self.source_path_long = os.path.join(self.long_path, '_test4.py')
shutil.copyfile(self.source_path, self.source_path_long)
self.bc_path_long = importlib.util.cache_from_source(
self.source_path_long
)

def tearDown(self):
shutil.rmtree(self.directory)
Expand Down Expand Up @@ -266,14 +256,22 @@ def test_compile_missing_multiprocessing(self, compile_file_mock):
compileall.compile_dir(self.directory, quiet=True, workers=5)
self.assertTrue(compile_file_mock.called)

def text_compile_dir_maxlevels(self):
# Test the actual impact of maxlevels attr
compileall.compile_dir(os.path.join(self.directory, "long"),
maxlevels=10, quiet=True)
self.assertFalse(os.path.isfile(self.bc_path_long))
compileall.compile_dir(os.path.join(self.directory, "long"),
maxlevels=110, quiet=True)
self.assertTrue(os.path.isfile(self.bc_path_long))
def test_compile_dir_maxlevels(self):
# Test the actual impact of maxlevels parameter
depth = 3
path = self.directory
for i in range(1, depth + 1):
path = os.path.join(path, "dir_{}".format(i))
source = os.path.join(path, 'script.py')
os.mkdir(path)
shutil.copyfile(self.source_path, source)
pyc_filename = importlib.util.cache_from_source(source)

compileall.compile_dir(self.directory, quiet=True, maxlevels=depth - 1)
self.assertFalse(os.path.isfile(pyc_filename))

compileall.compile_dir(self.directory, quiet=True, maxlevels=depth)
self.assertTrue(os.path.isfile(pyc_filename))

def test_strip_only(self):
fullpath = ["test", "build", "real", "path"]
Expand Down Expand Up @@ -330,6 +328,15 @@ def test_strip_and_prepend(self):
str(err, encoding=sys.getdefaultencoding())
)

def test_strip_prepend_and_ddir(self):
fullpath = ["test", "build", "real", "path", "ddir"]
path = os.path.join(self.directory, *fullpath)
os.makedirs(path)
script_helper.make_script(path, "test", "1 / 0")
with self.assertRaises(ValueError):
compileall.compile_dir(path, quiet=True, ddir="/bar",
stripdir="/foo", prependdir="/bar")

def test_multiple_optimization_levels(self):
script = script_helper.make_script(self.directory,
"test_optimization",
Expand All @@ -341,7 +348,7 @@ def test_multiple_optimization_levels(self):
test_combinations = [[0, 1], [1, 2], [0, 2], [0, 1, 2]]

for opt_combination in test_combinations:
compileall.compile_file(script,
compileall.compile_file(script, quiet=True,
optimize=opt_combination)
for opt_level in opt_combination:
self.assertTrue(os.path.isfile(bc[opt_level]))
Expand Down Expand Up @@ -372,7 +379,7 @@ def test_ignore_symlink_destination(self):
allowed_bc = importlib.util.cache_from_source(allowed_symlink)
prohibited_bc = importlib.util.cache_from_source(prohibited_symlink)

compileall.compile_dir(symlinks_path, quiet=False, limit_sl_dest=allowed_path)
compileall.compile_dir(symlinks_path, quiet=True, limit_sl_dest=allowed_path)

self.assertTrue(os.path.isfile(allowed_bc))
self.assertFalse(os.path.isfile(prohibited_bc))
Expand Down Expand Up @@ -642,21 +649,19 @@ def test_recursion_limit(self):
self.assertCompiled(spamfn)
self.assertCompiled(eggfn)

def test_default_recursion_limit(self):
many_directories = [str(number) for number in range(1, 100)]
self.long_path = os.path.join(self.directory,
"long",
*many_directories)
os.makedirs(self.long_path)
self.source_path_long = script_helper.make_script(
self.long_path, "deepscript", ""
)
self.bc_path_long = importlib.util.cache_from_source(
self.source_path_long
)
self.assertFalse(os.path.isfile(self.bc_path_long))
self.assertRunOK('-q', os.path.join(self.directory, "long"))
self.assertTrue(os.path.isfile(self.bc_path_long))
@support.skip_unless_symlink
def test_symlink_loop(self):
# Currently, compileall ignores symlinks to directories.
# If that limitation is ever lifted, it should protect against
# recursion in symlink loops.
pkg = os.path.join(self.pkgdir, 'spam')
script_helper.make_pkg(pkg)
os.symlink('.', os.path.join(pkg, 'evil'))
os.symlink('.', os.path.join(pkg, 'evil2'))
self.assertRunOK('-q', self.pkgdir)
self.assertCompiled(os.path.join(
self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py'
))

def test_quiet(self):
noisy = self.assertRunOK(self.pkgdir)
Expand Down