From 6a427060cdabadc36dfb577d54af6e91ae4672d1 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 26 Sep 2019 09:06:02 +0200 Subject: [PATCH 1/4] compileall: Skip long path path on Windows if the path can't be created --- Lib/test/test_compileall.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index af885c51224064..04c8d2c6919f8e 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -41,20 +41,34 @@ def setUp(self): os.mkdir(self.subdirectory) self.source_path3 = os.path.join(self.subdirectory, '_test3.py') shutil.copyfile(self.source_path, self.source_path3) + + def tearDown(self): + shutil.rmtree(self.directory) + + def create_long_path(self): many_directories = [str(number) for number in range(1, 100)] - self.long_path = os.path.join(self.directory, + long_path = os.path.join(self.directory, "long", *many_directories) - os.makedirs(self.long_path) - self.source_path_long = os.path.join(self.long_path, '_test4.py') + try: + os.makedirs(long_path) + except FileNotFoundError: + if sys.platform == "win32": + # On Windows, a FileNotFoundError("The filename or extension + # is too long") is raised for long paths + self.skipTest('Cannot create a long path') + raise + except OSError: + if exc.errno == errno.ENAMETOOLONG: + self.skipTest('Cannot create a long path') + else: + raise + self.source_path_long = os.path.join(self.long_path, '_test_long.py') shutil.copyfile(self.source_path, self.source_path_long) self.bc_path_long = importlib.util.cache_from_source( self.source_path_long ) - def tearDown(self): - shutil.rmtree(self.directory) - def add_bad_source_file(self): self.bad_source_path = os.path.join(self.directory, '_test_bad.py') with open(self.bad_source_path, 'w') as file: @@ -206,6 +220,7 @@ def test_compile_missing_multiprocessing(self, compile_file_mock): def text_compile_dir_maxlevels(self): # Test the actual impact of maxlevels attr + self.create_long_path() compileall.compile_dir(os.path.join(self.directory, "long"), maxlevels=10, quiet=True) self.assertFalse(os.path.isfile(self.bc_path_long)) From f0a3743487ec64f9dca7de5f9ef4dea582da8932 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 26 Sep 2019 09:45:52 +0200 Subject: [PATCH 2/4] Create a path as long as possible Also, actually run the test --- Lib/test/test_compileall.py | 42 ++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 04c8d2c6919f8e..d5bc5094b437d8 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -11,6 +11,7 @@ import time import unittest import io +import errno from unittest import mock, skipUnless try: @@ -46,24 +47,31 @@ def tearDown(self): shutil.rmtree(self.directory) def create_long_path(self): - many_directories = [str(number) for number in range(1, 100)] - long_path = os.path.join(self.directory, - "long", - *many_directories) - try: - os.makedirs(long_path) - except FileNotFoundError: - if sys.platform == "win32": + long_path = os.path.join(self.directory, "long") + os.makedirs(long_path) + + # Create a path as long as possible (with a limit) + for i in range(100): + long_path = os.path.join(long_path, f"long_directory_{i}") + try: + os.mkdir(long_path) + except FileNotFoundError: # On Windows, a FileNotFoundError("The filename or extension # is too long") is raised for long paths - self.skipTest('Cannot create a long path') - raise - except OSError: - if exc.errno == errno.ENAMETOOLONG: - self.skipTest('Cannot create a long path') - else: - raise - self.source_path_long = os.path.join(self.long_path, '_test_long.py') + if sys.platform != "win32": + raise + except OSError as exc: + if exc.errno != errno.ENAMETOOLONG: + raise + + print(i) + if i < 10: + raise ValueError('Path limit is too short') + + # Shorten the path again so the files fit + long_path = os.path.dirname(os.path.dirname(long_path)) + + self.source_path_long = os.path.join(long_path, '_test_long.py') shutil.copyfile(self.source_path, self.source_path_long) self.bc_path_long = importlib.util.cache_from_source( self.source_path_long @@ -218,7 +226,7 @@ def test_compile_missing_multiprocessing(self, compile_file_mock): compileall.compile_dir(self.directory, quiet=True, workers=5) self.assertTrue(compile_file_mock.called) - def text_compile_dir_maxlevels(self): + def test_compile_dir_maxlevels(self): # Test the actual impact of maxlevels attr self.create_long_path() compileall.compile_dir(os.path.join(self.directory, "long"), From 8680fab4dcc605acfb0c781af23518cb0b1f1d1d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 26 Sep 2019 10:09:32 +0200 Subject: [PATCH 3/4] Try creating directories (10 at a time) to get a long path --- Lib/test/test_compileall.py | 52 +++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index d5bc5094b437d8..38c2eff5376244 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -48,34 +48,46 @@ def tearDown(self): def create_long_path(self): long_path = os.path.join(self.directory, "long") - os.makedirs(long_path) - # Create a path as long as possible (with a limit) - for i in range(100): - long_path = os.path.join(long_path, f"long_directory_{i}") + # Create a long path, 10 directories at a time. + # It will be 100 directories deep, or shorter if the OS limits it. + for i in range(10): + longer_path = os.path.join( + long_path, *(f"long_directory_{i}_{j}" for j in range(10)) + ) + + # Check if we can open __pycache__/*.pyc. + # Also, put in the source file that we want to compile + longer_source = os.path.join(longer_path, '_test_long.py') + longer_cache = importlib.util.cache_from_source(longer_source) try: - os.mkdir(long_path) + os.makedirs(longer_path) + shutil.copyfile(self.source_path, longer_source) + os.makedirs(os.path.dirname(longer_cache)) + # Make sure we can write to the cache + with open(longer_cache, 'w'): + pass except FileNotFoundError: # On Windows, a FileNotFoundError("The filename or extension # is too long") is raised for long paths - if sys.platform != "win32": - raise + if sys.platform == "win32": + break except OSError as exc: - if exc.errno != errno.ENAMETOOLONG: - raise + if exc.errno == errno.ENAMETOOLONG: + break - print(i) - if i < 10: - raise ValueError('Path limit is too short') + # Remove the __pycache__ + shutil.rmtree(os.path.dirname(longer_cache)) - # Shorten the path again so the files fit - long_path = os.path.dirname(os.path.dirname(long_path)) + long_path = longer_path + long_source = longer_source + long_cache = longer_cache - self.source_path_long = os.path.join(long_path, '_test_long.py') - shutil.copyfile(self.source_path, self.source_path_long) - self.bc_path_long = importlib.util.cache_from_source( - self.source_path_long - ) + if i < 2: + raise ValueError('Path limit is too short') + + self.source_path_long = long_source + self.bc_path_long = long_cache def add_bad_source_file(self): self.bad_source_path = os.path.join(self.directory, '_test_bad.py') @@ -233,7 +245,7 @@ def test_compile_dir_maxlevels(self): maxlevels=10, quiet=True) self.assertFalse(os.path.isfile(self.bc_path_long)) compileall.compile_dir(os.path.join(self.directory, "long"), - maxlevels=110, quiet=True) + quiet=True) self.assertTrue(os.path.isfile(self.bc_path_long)) def test_strip_only(self): From 18496318ca2a73049668dd5a2f7b99d3c4fc9697 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 26 Sep 2019 11:24:19 +0200 Subject: [PATCH 4/4] Raise on unexpected errors --- Lib/test/test_compileall.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 38c2eff5376244..205457f3b4801e 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -72,9 +72,13 @@ def create_long_path(self): # is too long") is raised for long paths if sys.platform == "win32": break + else: + raise except OSError as exc: if exc.errno == errno.ENAMETOOLONG: break + else: + raise # Remove the __pycache__ shutil.rmtree(os.path.dirname(longer_cache))