From ee16c91e37004b33a97bff463bbcf5558915616d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 11 Nov 2021 13:16:33 -0700 Subject: [PATCH 01/12] Use git status --porcelain. --- Tools/freeze/test/freeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 387f1ff70b234c..1ed25a2ba9563e 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -92,7 +92,7 @@ def git_copy_repo(newroot, oldroot): _run_quiet([GIT, 'clone', oldroot, newroot]) # Copy over any uncommited files. - text = _run_stdout([GIT, 'status', '-s'], oldroot) + text = _run_stdout([GIT, 'status', '--porcelain=1'], oldroot) for line in text.splitlines(): _, _, relfile = line.strip().partition(' ') relfile = relfile.strip() From ca38a9b97e06852718cb85ac9788f97f74395a94 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 11 Nov 2021 13:36:17 -0700 Subject: [PATCH 02/12] Clean up ignored files too. --- Tools/freeze/test/freeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 1ed25a2ba9563e..50cb4a10910453 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -83,7 +83,7 @@ def git_copy_repo(newroot, oldroot): print(f'updating copied repo {newroot}...') if newroot == SRCDIR: raise Exception('this probably isn\'t what you wanted') - _run_quiet([GIT, 'clean', '-d', '-f'], newroot) + _run_quiet([GIT, 'clean', '-d', '-x', '--force'], newroot) _run_quiet([GIT, 'reset'], newroot) _run_quiet([GIT, 'checkout', '.'], newroot) _run_quiet([GIT, 'pull', '-f', oldroot], newroot) From 2fe7750882c887a787f75e144c13644c62e2ac4b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 11 Nov 2021 13:40:47 -0700 Subject: [PATCH 03/12] Properly clean up the repo copy. --- Tools/freeze/test/freeze.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 50cb4a10910453..49ce538149fd40 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -83,10 +83,10 @@ def git_copy_repo(newroot, oldroot): print(f'updating copied repo {newroot}...') if newroot == SRCDIR: raise Exception('this probably isn\'t what you wanted') + rev = _run_stdout([GIT, 'rev-parse', 'HEAD'], newroot) _run_quiet([GIT, 'clean', '-d', '-x', '--force'], newroot) - _run_quiet([GIT, 'reset'], newroot) - _run_quiet([GIT, 'checkout', '.'], newroot) - _run_quiet([GIT, 'pull', '-f', oldroot], newroot) + _run_quiet([GIT, 'fetch', oldroot], newroot) + _run_quiet([GIT, 'reset', '--hard', rev], newroot) else: print(f'copying repo into {newroot}...') _run_quiet([GIT, 'clone', oldroot, newroot]) From ba62c88d528c9352d03f147cf09aa5c885c624e1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 11 Nov 2021 13:42:07 -0700 Subject: [PATCH 04/12] Handle moved files. --- Tools/freeze/test/freeze.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 49ce538149fd40..c149a40d14bb44 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -94,7 +94,8 @@ def git_copy_repo(newroot, oldroot): # Copy over any uncommited files. text = _run_stdout([GIT, 'status', '--porcelain=1'], oldroot) for line in text.splitlines(): - _, _, relfile = line.strip().partition(' ') + _, _, loc = line.strip().partition(' ') + _, _, relfile = loc.rpartition(' -> ') relfile = relfile.strip() isdir = relfile.endswith(os.path.sep) relfile = relfile.rstrip(os.path.sep) From 7cbdd2c0075f71168feeb7911eed34ebeec47259 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 11 Nov 2021 13:42:28 -0700 Subject: [PATCH 05/12] Handle deleted files. --- Tools/freeze/test/freeze.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index c149a40d14bb44..329d193f1a11f3 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -101,11 +101,15 @@ def git_copy_repo(newroot, oldroot): relfile = relfile.rstrip(os.path.sep) srcfile = os.path.join(oldroot, relfile) dstfile = os.path.join(newroot, relfile) - os.makedirs(os.path.dirname(dstfile), exist_ok=True) - if isdir: - shutil.copytree(srcfile, dstfile, dirs_exist_ok=True) + if os.path.exists(srcfile): + os.makedirs(os.path.dirname(dstfile), exist_ok=True) + if isdir: + shutil.copytree(srcfile, dstfile, dirs_exist_ok=True) + else: + shutil.copy2(srcfile, dstfile) else: - shutil.copy2(srcfile, dstfile) + # Only single files will show up here. + os.unlink(dstfile) def get_makefile_var(builddir, name): From 63f02a634224b2d5ac6ac870262b3ddb40e3cfcd Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 11 Nov 2021 13:54:05 -0700 Subject: [PATCH 06/12] Print out which files are involved. --- Tools/freeze/test/freeze.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 329d193f1a11f3..04d0b95ccbc805 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -151,6 +151,7 @@ def prepare(script=None, outdir=None): # Write the script to disk. if script: scriptfile = os.path.join(outdir, 'app.py') + print(f'creating the script to be frozen at {scriptfile}') with open(scriptfile, 'w') as outfile: outfile.write(script) @@ -177,7 +178,7 @@ def prepare(script=None, outdir=None): raise UnsupportedError('make') # Build python. - print('building python...') + print(f'building python in {builddir}...') if os.path.exists(os.path.join(srcdir, 'Makefile')): # Out-of-tree builds require a clean srcdir. _run_quiet([MAKE, '-C', srcdir, 'clean']) From e6a2453de185928c6c32a9af5dd57cb97fec2b9f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 11 Nov 2021 13:56:17 -0700 Subject: [PATCH 07/12] Add a NEWS entry. --- Misc/NEWS.d/next/Tests/2021-11-11-13-56-00.bpo-45783.8k1Rng.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Tests/2021-11-11-13-56-00.bpo-45783.8k1Rng.rst diff --git a/Misc/NEWS.d/next/Tests/2021-11-11-13-56-00.bpo-45783.8k1Rng.rst b/Misc/NEWS.d/next/Tests/2021-11-11-13-56-00.bpo-45783.8k1Rng.rst new file mode 100644 index 00000000000000..cb0e07f27fb553 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2021-11-11-13-56-00.bpo-45783.8k1Rng.rst @@ -0,0 +1 @@ +The test for the freeze tool now handles file moves and deletions. From bd20dc908c0b54082f861d4083939db658afe3cc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 11 Nov 2021 13:59:44 -0700 Subject: [PATCH 08/12] Also delete directories. --- Tools/freeze/test/freeze.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 04d0b95ccbc805..ae92b6a38ba43d 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -108,8 +108,10 @@ def git_copy_repo(newroot, oldroot): else: shutil.copy2(srcfile, dstfile) else: - # Only single files will show up here. - os.unlink(dstfile) + if isdir: + shutil.rmtree(dstfile) + else: + os.unlink(dstfile) def get_makefile_var(builddir, name): From 5aa84546177b3d39d94fcdc74a58a4ab0e309050 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 12 Nov 2021 10:11:12 -0700 Subject: [PATCH 09/12] Use a temp dir. --- Lib/test/test_tools/test_freeze.py | 9 +++++---- Tools/freeze/test/freeze.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py index 392a776f042e4f..386c35a973bc27 100644 --- a/Lib/test/test_tools/test_freeze.py +++ b/Lib/test/test_tools/test_freeze.py @@ -5,6 +5,7 @@ import unittest from test import support +from test.support import os_helper from . import imports_under_tool, skip_if_missing skip_if_missing('freeze') @@ -22,8 +23,8 @@ def test_freeze_simple_script(self): print('running...') sys.exit(0) """) - outdir, scriptfile, python = helper.prepare(script) - - executable = helper.freeze(python, scriptfile, outdir) - text = helper.run(executable) + with os_helper.temp_dir() as outdir: + outdir, scriptfile, python = helper.prepare(script, outdir) + executable = helper.freeze(python, scriptfile, outdir) + text = helper.run(executable) self.assertEqual(text, 'running...') diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index ae92b6a38ba43d..a8a7f23789a4b9 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -157,7 +157,8 @@ def prepare(script=None, outdir=None): with open(scriptfile, 'w') as outfile: outfile.write(script) - # Make a copy of the repo to avoid affecting the current build. + # Make a copy of the repo to avoid affecting the current build + # (e.g. changing PREFIX). srcdir = os.path.join(outdir, 'cpython') git_copy_repo(srcdir, SRCDIR) From f10bc26c1065525032f44c2369be0ea91ade0c78 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 12 Nov 2021 11:24:01 -0700 Subject: [PATCH 10/12] Do the right thing if not a git repo. --- Tools/freeze/test/freeze.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index a8a7f23789a4b9..b78befa12e2d4b 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -75,6 +75,16 @@ def ensure_opt(args, name, value): args[pos] = f'{opt}={value}' +def copy_source_tree(newroot, oldroot): + print(f'copying the source tree into {newroot}...') + if os.path.exists(newroot): + if newroot == SRCDIR: + raise Exception('this probably isn\'t what you wanted') + shutil.rmtree(newroot) + shutil.copytree(oldroot, newroot) + _run_quiet([MAKE, 'clean'], newroot) + + def git_copy_repo(newroot, oldroot): if not GIT: raise UnsupportedError('git') @@ -160,7 +170,10 @@ def prepare(script=None, outdir=None): # Make a copy of the repo to avoid affecting the current build # (e.g. changing PREFIX). srcdir = os.path.join(outdir, 'cpython') - git_copy_repo(srcdir, SRCDIR) + if os.path.exists(os.path.join(SRCDIR, '.git')): + git_copy_repo(srcdir, SRCDIR) + else: + copy_source_tree(srcdir, SRCDIR) # We use an out-of-tree build (instead of srcdir). builddir = os.path.join(outdir, 'python-build') From fea9a4efa5c2590d1210c161e42b7c14d3a60f43 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 12 Nov 2021 11:26:59 -0700 Subject: [PATCH 11/12] Do not bother with git. --- Tools/freeze/test/freeze.py | 45 +------------------------------------ 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index b78befa12e2d4b..809c71e2fa39c4 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -11,7 +11,6 @@ SRCDIR = os.path.dirname(os.path.dirname(TOOL_ROOT)) MAKE = shutil.which('make') -GIT = shutil.which('git') FREEZE = os.path.join(TOOL_ROOT, 'freeze.py') OUTDIR = os.path.join(TESTS_DIR, 'outdir') @@ -85,45 +84,6 @@ def copy_source_tree(newroot, oldroot): _run_quiet([MAKE, 'clean'], newroot) -def git_copy_repo(newroot, oldroot): - if not GIT: - raise UnsupportedError('git') - - if os.path.exists(newroot): - print(f'updating copied repo {newroot}...') - if newroot == SRCDIR: - raise Exception('this probably isn\'t what you wanted') - rev = _run_stdout([GIT, 'rev-parse', 'HEAD'], newroot) - _run_quiet([GIT, 'clean', '-d', '-x', '--force'], newroot) - _run_quiet([GIT, 'fetch', oldroot], newroot) - _run_quiet([GIT, 'reset', '--hard', rev], newroot) - else: - print(f'copying repo into {newroot}...') - _run_quiet([GIT, 'clone', oldroot, newroot]) - - # Copy over any uncommited files. - text = _run_stdout([GIT, 'status', '--porcelain=1'], oldroot) - for line in text.splitlines(): - _, _, loc = line.strip().partition(' ') - _, _, relfile = loc.rpartition(' -> ') - relfile = relfile.strip() - isdir = relfile.endswith(os.path.sep) - relfile = relfile.rstrip(os.path.sep) - srcfile = os.path.join(oldroot, relfile) - dstfile = os.path.join(newroot, relfile) - if os.path.exists(srcfile): - os.makedirs(os.path.dirname(dstfile), exist_ok=True) - if isdir: - shutil.copytree(srcfile, dstfile, dirs_exist_ok=True) - else: - shutil.copy2(srcfile, dstfile) - else: - if isdir: - shutil.rmtree(dstfile) - else: - os.unlink(dstfile) - - def get_makefile_var(builddir, name): regex = re.compile(rf'^{name} *=\s*(.*?)\s*$') filename = os.path.join(builddir, 'Makefile') @@ -170,10 +130,7 @@ def prepare(script=None, outdir=None): # Make a copy of the repo to avoid affecting the current build # (e.g. changing PREFIX). srcdir = os.path.join(outdir, 'cpython') - if os.path.exists(os.path.join(SRCDIR, '.git')): - git_copy_repo(srcdir, SRCDIR) - else: - copy_source_tree(srcdir, SRCDIR) + copy_source_tree(srcdir, SRCDIR) # We use an out-of-tree build (instead of srcdir). builddir = os.path.join(outdir, 'python-build') From fb1b34e946967cd3ac681397000221b19fbd4f85 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 16 Nov 2021 09:12:52 -0700 Subject: [PATCH 12/12] Do not run "make clean" if there is no Makefile. --- Tools/freeze/test/freeze.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index 809c71e2fa39c4..db2aa3101d8790 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -81,7 +81,8 @@ def copy_source_tree(newroot, oldroot): raise Exception('this probably isn\'t what you wanted') shutil.rmtree(newroot) shutil.copytree(oldroot, newroot) - _run_quiet([MAKE, 'clean'], newroot) + if os.path.exists(os.path.join(newroot, 'Makefile')): + _run_quiet([MAKE, 'clean'], newroot) def get_makefile_var(builddir, name):