From b1cbcc1c71748b6cb32953886d4b5c73d7ea3cdf Mon Sep 17 00:00:00 2001 From: Patrik Kopkan Date: Thu, 25 Apr 2019 11:15:07 +0200 Subject: [PATCH 1/8] bpo-37064: add option -k to Tools/scripts/pathfix.py - with this option it is possible to keep flags when changing shebang line --- Lib/test/test_tools/test_pathfix.py | 34 ++++++++++++++++ .../2019-05-27-15-26-12.bpo-37064.k_SPW2.rst | 1 + Tools/scripts/pathfix.py | 40 +++++++++++++++++-- 3 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 Lib/test/test_tools/test_pathfix.py create mode 100644 Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py new file mode 100644 index 00000000000000..12e8c1035adbe6 --- /dev/null +++ b/Lib/test/test_tools/test_pathfix.py @@ -0,0 +1,34 @@ +import unittest, os, subprocess +from test.test_tools import import_tool, scriptsdir +import sys +from test import support + + +class TestPathfixFunctional(unittest.TestCase): + script = os.path.join(scriptsdir, 'pathfix.py') + + @classmethod + def setupUpClass(cls): + self.addCleanup(support.unlink, support.TESTFN) + + def setUp(self): + self.temp_file = support.TESTFN + + with open(self.temp_file, 'w') as f: + f.write('#! /usr/bin/env python -R\n' + 'print("Hello world")\n') + + def test_pathfix_keeping_flags(self): + subprocess.call([sys.executable, self.script, '-i', '/usr/bin/python', '-k', '-n', self.temp_file]) + with open(self.temp_file) as f: + output = f.read() + self.assertEqual(output, '#! /usr/bin/python -R\n' + 'print("Hello world")\n') + + def test_pathfix_without_keeping_flags(self): + subprocess.call([sys.executable, self.script, '-i', '/usr/bin/python', '-n', self.temp_file]) + with open(self.temp_file) as f: + output = f.read() + self.assertEqual(output, '#! /usr/bin/python\n' + 'print("Hello world")\n') + + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst new file mode 100644 index 00000000000000..46cf9bbac1081e --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst @@ -0,0 +1 @@ +Add flag -k to pathscript.py. This flag preserve flags from shebang before change and adds them after change. \ No newline at end of file diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 1a0cf1c9e69ce7..5b5c53dfdd2af4 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Change the #! line occurring in Python scripts. The new interpreter +# Change the #! line (shebang) occurring in Python scripts. The new interpreter # pathname must be given with a -i option. # # Command line arguments are files or directories to be processed. @@ -10,7 +10,12 @@ # arguments). # The original file is kept as a back-up (with a "~" attached to its name), # -n flag can be used to disable this. -# + +# Sometimes you may find shebangs with flags such as `#! /usr/bin/env python -si`. +# Normally, pathfix overwrites the entire line, including the flags. +# To change interpreter and keep flags from the original shebang line, use -k. + + # Undoubtedly you can do this using find and sed or perl, but this is # a nice example of Python code that recurses down a directory tree # and uses regular expressions. Also note several subtleties like @@ -33,16 +38,20 @@ new_interpreter = None preserve_timestamps = False create_backup = True +add_flag = None +keep_flags = False def main(): global new_interpreter global preserve_timestamps global create_backup - usage = ('usage: %s -i /interpreter -p -n file-or-directory ...\n' % + global keep_flags + + usage = ('usage: %s -i /interpreter -p -n -k file-or-directory ...\n' % sys.argv[0]) try: - opts, args = getopt.getopt(sys.argv[1:], 'i:pn') + opts, args = getopt.getopt(sys.argv[1:], 'i:kpn') except getopt.error as msg: err(str(msg) + '\n') err(usage) @@ -54,6 +63,8 @@ def main(): preserve_timestamps = True if o == '-n': create_backup = False + if o == '-k': + keep_flags = True if not new_interpreter or not new_interpreter.startswith(b'/') or \ not args: err('-i option or file-or-directory missing\n') @@ -70,10 +81,14 @@ def main(): if fix(arg): bad = 1 sys.exit(bad) + ispythonprog = re.compile(r'^[a-zA-Z0-9_]+\.py$') + + def ispython(name): return bool(ispythonprog.match(name)) + def recursedown(dirname): dbg('recursedown(%r)\n' % (dirname,)) bad = 0 @@ -96,6 +111,7 @@ def recursedown(dirname): if recursedown(fullname): bad = 1 return bad + def fix(filename): ## dbg('fix(%r)\n' % (filename,)) try: @@ -164,12 +180,28 @@ def fix(filename): # Return success return 0 + +def parse_shebang(shebangline): + end = len(shebangline) + start = shebangline.find(b' -', 0, end) # .find() returns index at space + if start == -1: + return b'', b'' + print(shebangline[(start + 2):end]) + return shebangline[(start + 2):end] + + def fixline(line): + global keep_flags if not line.startswith(b'#!'): return line + if b"python" not in line: return line + if keep_flags: + flags = parse_shebang(line) + return b'#! ' + new_interpreter + b" -" + flags return b'#! ' + new_interpreter + b'\n' + if __name__ == '__main__': main() From 48e07a3edc2519563c0476031d5af4a778be1776 Mon Sep 17 00:00:00 2001 From: Patrik Kopkan Date: Mon, 2 Sep 2019 16:52:34 +0200 Subject: [PATCH 2/8] - add helper test function --- Lib/test/test_tools/test_pathfix.py | 42 ++++++++++++------- .../2019-05-27-15-26-12.bpo-37064.k_SPW2.rst | 2 +- Tools/scripts/pathfix.py | 5 +-- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py index 12e8c1035adbe6..a520be970a33ef 100644 --- a/Lib/test/test_tools/test_pathfix.py +++ b/Lib/test/test_tools/test_pathfix.py @@ -7,28 +7,42 @@ class TestPathfixFunctional(unittest.TestCase): script = os.path.join(scriptsdir, 'pathfix.py') - @classmethod - def setupUpClass(cls): - self.addCleanup(support.unlink, support.TESTFN) - def setUp(self): self.temp_file = support.TESTFN + self.addCleanup(support.unlink, support.TESTFN) + def pathfix(self, shebang, pathfix_flags): with open(self.temp_file, 'w') as f: - f.write('#! /usr/bin/env python -R\n' + 'print("Hello world")\n') - - def test_pathfix_keeping_flags(self): - subprocess.call([sys.executable, self.script, '-i', '/usr/bin/python', '-k', '-n', self.temp_file]) + f.write(f'{shebang}\n' + 'print("Hello world")\n') + subprocess.call([sys.executable, self.script, *pathfix_flags, self.temp_file]) with open(self.temp_file) as f: output = f.read() - self.assertEqual(output, '#! /usr/bin/python -R\n' + 'print("Hello world")\n') + print(output) + return output - def test_pathfix_without_keeping_flags(self): - subprocess.call([sys.executable, self.script, '-i', '/usr/bin/python', '-n', self.temp_file]) - with open(self.temp_file) as f: - output = f.read() - self.assertEqual(output, '#! /usr/bin/python\n' + 'print("Hello world")\n') + def test_pathfix_keeping_flags(self): + self.assertEqual( + self.pathfix( + '#! /usr/bin/env python -R', + ['-i', '/usr/bin/python', '-k', '-n']), + '#! /usr/bin/python -R\n' + 'print("Hello world")\n' + ) + def test_pathfix_without_keeping_flags(self): + self.assertEqual( + self.pathfix( + '#! /usr/bin/env python -R', + ['-i', '/usr/bin/python', '-n']), + '#! /usr/bin/python\n' + 'print("Hello world")\n' + ) + + def test_pathfix(self): + self.assertEqual( + self.pathfix( + '#! /usr/bin/env python', + ['-i', '/usr/bin/python', '-n']), + '#! /usr/bin/python\n' + 'print("Hello world")\n' + ) if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst index 46cf9bbac1081e..02599077c9d5f9 100644 --- a/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst +++ b/Misc/NEWS.d/next/Tools-Demos/2019-05-27-15-26-12.bpo-37064.k_SPW2.rst @@ -1 +1 @@ -Add flag -k to pathscript.py. This flag preserve flags from shebang before change and adds them after change. \ No newline at end of file +Add flag -k to pathscript.py script: preserve shebang flags. \ No newline at end of file diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 5b5c53dfdd2af4..3222612b3b9b55 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -38,7 +38,6 @@ new_interpreter = None preserve_timestamps = False create_backup = True -add_flag = None keep_flags = False @@ -183,15 +182,13 @@ def fix(filename): def parse_shebang(shebangline): end = len(shebangline) - start = shebangline.find(b' -', 0, end) # .find() returns index at space + start = shebangline.find(b' -', 0, end) if start == -1: return b'', b'' - print(shebangline[(start + 2):end]) return shebangline[(start + 2):end] def fixline(line): - global keep_flags if not line.startswith(b'#!'): return line From 7f5fcbc9b24d5930cf6b0da20e7a005f853f8a28 Mon Sep 17 00:00:00 2001 From: Patrik Kopkan Date: Mon, 2 Sep 2019 17:00:32 +0200 Subject: [PATCH 3/8] strip shebang in parse_shebang --- Tools/scripts/pathfix.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 3222612b3b9b55..8585b178cf9ed5 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -181,6 +181,7 @@ def fix(filename): def parse_shebang(shebangline): + shebangline = shebangline.strip() end = len(shebangline) start = shebangline.find(b' -', 0, end) if start == -1: @@ -196,7 +197,7 @@ def fixline(line): return line if keep_flags: flags = parse_shebang(line) - return b'#! ' + new_interpreter + b" -" + flags + return b'#! ' + new_interpreter + b' -' + flags + b'\n' return b'#! ' + new_interpreter + b'\n' From 25d361f0778f38fb295a8cfc73195d6f27b71431 Mon Sep 17 00:00:00 2001 From: Patrik Kopkan Date: Thu, 5 Sep 2019 11:02:28 +0200 Subject: [PATCH 4/8] little fixes --- Lib/test/test_tools/test_pathfix.py | 30 ++++++++++++++++------------- Tools/scripts/pathfix.py | 2 +- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py index a520be970a33ef..6bf80553703437 100644 --- a/Lib/test/test_tools/test_pathfix.py +++ b/Lib/test/test_tools/test_pathfix.py @@ -1,7 +1,9 @@ -import unittest, os, subprocess -from test.test_tools import import_tool, scriptsdir +import os +import subprocess import sys +import unittest from test import support +from test.test_tools import import_tool, scriptsdir class TestPathfixFunctional(unittest.TestCase): @@ -12,36 +14,38 @@ def setUp(self): self.addCleanup(support.unlink, support.TESTFN) def pathfix(self, shebang, pathfix_flags): - with open(self.temp_file, 'w') as f: + with open(self.temp_file, 'w', encoding='utf8') as f: f.write(f'{shebang}\n' + 'print("Hello world")\n') - subprocess.call([sys.executable, self.script, *pathfix_flags, self.temp_file]) - with open(self.temp_file) as f: + subprocess.call([sys.executable, self.script, *pathfix_flags, '-n', self.temp_file]) + with open(self.temp_file, 'r', encoding='utf8') as f: output = f.read() - print(output) - return output + lines = output.split('\n') + self.assertEqual(lines[1:], ['print("Hello world")\n'] + shebang = lines[0][:-1] # strip the newline + return shebang def test_pathfix_keeping_flags(self): self.assertEqual( self.pathfix( '#! /usr/bin/env python -R', - ['-i', '/usr/bin/python', '-k', '-n']), - '#! /usr/bin/python -R\n' + 'print("Hello world")\n' + ['-i', '/usr/bin/python3', '-k',]), + '#! /usr/bin/python3 -R\n', ) def test_pathfix_without_keeping_flags(self): self.assertEqual( self.pathfix( '#! /usr/bin/env python -R', - ['-i', '/usr/bin/python', '-n']), - '#! /usr/bin/python\n' + 'print("Hello world")\n' + ['-i', '/usr/bin/python3',]), + '#! /usr/bin/python3\n', ) def test_pathfix(self): self.assertEqual( self.pathfix( '#! /usr/bin/env python', - ['-i', '/usr/bin/python', '-n']), - '#! /usr/bin/python\n' + 'print("Hello world")\n' + ['-i', '/usr/bin/python3',]), + '#! /usr/bin/python3\n', ) if __name__ == '__main__': diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 8585b178cf9ed5..f375f975a3925f 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -185,7 +185,7 @@ def parse_shebang(shebangline): end = len(shebangline) start = shebangline.find(b' -', 0, end) if start == -1: - return b'', b'' + return b'' return shebangline[(start + 2):end] From f06281729a426810492ec5abcdc02554f2db7d5b Mon Sep 17 00:00:00 2001 From: Patrik Kopkan Date: Thu, 5 Sep 2019 16:00:56 +0200 Subject: [PATCH 5/8] 'adress Victor's comments' --- Lib/test/test_tools/test_pathfix.py | 30 +++++++++++++++++------------ Tools/scripts/pathfix.py | 9 ++++----- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py index 6bf80553703437..8f2a018322cf3d 100644 --- a/Lib/test/test_tools/test_pathfix.py +++ b/Lib/test/test_tools/test_pathfix.py @@ -20,33 +20,39 @@ def pathfix(self, shebang, pathfix_flags): with open(self.temp_file, 'r', encoding='utf8') as f: output = f.read() lines = output.split('\n') - self.assertEqual(lines[1:], ['print("Hello world")\n'] - shebang = lines[0][:-1] # strip the newline + self.assertEqual(lines[1:], ['print("Hello world")', '']) + shebang = lines[0] return shebang - def test_pathfix_keeping_flags(self): + def test_pathfix(self): + self.assertEqual( + self.pathfix( + '#! /usr/bin/env python', + ['-i', '/usr/bin/python3',]), + '#! /usr/bin/python3', + ) self.assertEqual( self.pathfix( '#! /usr/bin/env python -R', - ['-i', '/usr/bin/python3', '-k',]), - '#! /usr/bin/python3 -R\n', + ['-i', '/usr/bin/python3', ]), + '#! /usr/bin/python3', ) - def test_pathfix_without_keeping_flags(self): + def test_pathfix_keeping_flags(self): self.assertEqual( self.pathfix( '#! /usr/bin/env python -R', - ['-i', '/usr/bin/python3',]), - '#! /usr/bin/python3\n', + ['-i', '/usr/bin/python3', '-k',]), + '#! /usr/bin/python3 -R', ) - - def test_pathfix(self): self.assertEqual( self.pathfix( '#! /usr/bin/env python', - ['-i', '/usr/bin/python3',]), - '#! /usr/bin/python3\n', + ['-i', '/usr/bin/python3', '-k',]), + '#! /usr/bin/python3', ) + + if __name__ == '__main__': unittest.main() diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index f375f975a3925f..1bd416cddb162f 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -182,11 +182,10 @@ def fix(filename): def parse_shebang(shebangline): shebangline = shebangline.strip() - end = len(shebangline) - start = shebangline.find(b' -', 0, end) + start = shebangline.find(b' -') if start == -1: return b'' - return shebangline[(start + 2):end] + return shebangline[(start):] def fixline(line): @@ -195,10 +194,10 @@ def fixline(line): if b"python" not in line: return line + flags = b'' if keep_flags: flags = parse_shebang(line) - return b'#! ' + new_interpreter + b' -' + flags + b'\n' - return b'#! ' + new_interpreter + b'\n' + return b'#! ' + new_interpreter + flags + b'\n' if __name__ == '__main__': From 0a5d298a1035c6a3b207be48eca2bdd284828a9e Mon Sep 17 00:00:00 2001 From: Patrik Kopkan Date: Thu, 5 Sep 2019 16:14:15 +0200 Subject: [PATCH 6/8] check exit codes --- Lib/test/test_tools/test_pathfix.py | 6 +++++- Tools/scripts/pathfix.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py index 8f2a018322cf3d..162280abfb05fe 100644 --- a/Lib/test/test_tools/test_pathfix.py +++ b/Lib/test/test_tools/test_pathfix.py @@ -16,7 +16,11 @@ def setUp(self): def pathfix(self, shebang, pathfix_flags): with open(self.temp_file, 'w', encoding='utf8') as f: f.write(f'{shebang}\n' + 'print("Hello world")\n') - subprocess.call([sys.executable, self.script, *pathfix_flags, '-n', self.temp_file]) + proc = subprocess.run( + [sys.executable, self.script, + *pathfix_flags, '-n', self.temp_file], + capture_output=True) + self.assertEqual(proc.returncode, 0, proc) with open(self.temp_file, 'r', encoding='utf8') as f: output = f.read() lines = output.split('\n') diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 1bd416cddb162f..eb8b519b0f0ddf 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -185,7 +185,7 @@ def parse_shebang(shebangline): start = shebangline.find(b' -') if start == -1: return b'' - return shebangline[(start):] + return shebangline[start:] def fixline(line): From 6e79ff6ab8dcaa246dfde9bb1c152250ab103425 Mon Sep 17 00:00:00 2001 From: Patrik Kopkan Date: Thu, 5 Sep 2019 16:17:40 +0200 Subject: [PATCH 7/8] cleanup --- Lib/test/test_tools/test_pathfix.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py index 162280abfb05fe..5c0dd1d5a60c06 100644 --- a/Lib/test/test_tools/test_pathfix.py +++ b/Lib/test/test_tools/test_pathfix.py @@ -16,17 +16,20 @@ def setUp(self): def pathfix(self, shebang, pathfix_flags): with open(self.temp_file, 'w', encoding='utf8') as f: f.write(f'{shebang}\n' + 'print("Hello world")\n') + proc = subprocess.run( [sys.executable, self.script, *pathfix_flags, '-n', self.temp_file], capture_output=True) self.assertEqual(proc.returncode, 0, proc) + with open(self.temp_file, 'r', encoding='utf8') as f: output = f.read() - lines = output.split('\n') - self.assertEqual(lines[1:], ['print("Hello world")', '']) - shebang = lines[0] - return shebang + + lines = output.split('\n') + self.assertEqual(lines[1:], ['print("Hello world")', '']) + shebang = lines[0] + return shebang def test_pathfix(self): self.assertEqual( @@ -57,6 +60,5 @@ def test_pathfix_keeping_flags(self): ) - if __name__ == '__main__': unittest.main() From 98612cf3e685bfa0a056e823252c7ebc55156d7f Mon Sep 17 00:00:00 2001 From: Patrik Kopkan Date: Thu, 5 Sep 2019 16:28:30 +0200 Subject: [PATCH 8/8] use rstrip --- Tools/scripts/pathfix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index eb8b519b0f0ddf..eee684e3502d5b 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -181,7 +181,7 @@ def fix(filename): def parse_shebang(shebangline): - shebangline = shebangline.strip() + shebangline = shebangline.rstrip(b'\n') start = shebangline.find(b' -') if start == -1: return b''