From b5e8eb5365095c100fe80db969d2d4cd6eca9b9a Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 19 Oct 2015 13:27:41 +0300 Subject: [PATCH 1/9] Code to preserve original linefeeds (issue #121) --- libmodernize/main.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/libmodernize/main.py b/libmodernize/main.py index 4fe934c..95466e3 100644 --- a/libmodernize/main.py +++ b/libmodernize/main.py @@ -17,6 +17,35 @@ from libmodernize import __version__ from libmodernize.fixes import lib2to3_fix_names, six_fix_names, opt_in_fix_names + +class LFPreservingRefactoringTool(StdoutRefactoringTool): + """ https://github.com/python-modernize/python-modernize/issues/121 """ + def write_file(self, new_text, filename, old_text, encoding): + # detect linefeeds + lineends = {'\n':0, '\r\n':0, '\r':0} + lines = [] + for line in open(filename, 'rb'): + if line.endswith('\r\n'): + lineends['\r\n'] += 1 + elif line.endswith('\n'): + lineends['\n'] += 1 + elif line.endswith('\r'): + lineends['\r'] += 1 + lines.append(line.rstrip(\r\n)) + super(LFPreservingRefactoringTool, self).write_file( + new_text, filename, old_text, encoding) + # detect if line ends are consistent in source file + if sum([bool(lineends[x]) for x in lineends]) == 1: + # detect if line ends are different from system-specific + newline = [x for x in lineends if lineends[x] != 0][0] + if os.linesep != newline: + with open(filename, 'wb') as f: + for line in lines: + f.write(line) + self.log_debug('fixed %s linefeeds back to %s', + filename, newline) + + usage = __doc__ + """\ %s @@ -125,7 +154,7 @@ def main(args=None): else: requested = default_fixes fixer_names = requested.difference(unwanted_fixes) - rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit), + rt = LFPreservingRefactoringTool(sorted(fixer_names), flags, sorted(explicit), options.nobackups, not options.no_diffs) # Refactor all files and directories passed as arguments From d4109952cb48abab04a19e0d68676a4e4f35471e Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 19 Oct 2015 13:29:44 +0300 Subject: [PATCH 2/9] Typos --- libmodernize/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmodernize/main.py b/libmodernize/main.py index 95466e3..b5ffe21 100644 --- a/libmodernize/main.py +++ b/libmodernize/main.py @@ -31,7 +31,7 @@ def write_file(self, new_text, filename, old_text, encoding): lineends['\n'] += 1 elif line.endswith('\r'): lineends['\r'] += 1 - lines.append(line.rstrip(\r\n)) + lines.append(line.rstrip('\r\n')) super(LFPreservingRefactoringTool, self).write_file( new_text, filename, old_text, encoding) # detect if line ends are consistent in source file From cb502aec62362137cb064ffad4e022cbd9974414 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 19 Oct 2015 13:33:33 +0300 Subject: [PATCH 3/9] import os is needed --- libmodernize/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libmodernize/main.py b/libmodernize/main.py index b5ffe21..4128332 100644 --- a/libmodernize/main.py +++ b/libmodernize/main.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, print_function +import os import sys import logging import optparse From 7787615383522cc8389f4169a3be973ab468ca16 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 19 Oct 2015 13:51:38 +0300 Subject: [PATCH 4/9] Make sure to write new_text on rewriting newlines Thanks @takluyver for review --- libmodernize/main.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libmodernize/main.py b/libmodernize/main.py index 4128332..35f99f2 100644 --- a/libmodernize/main.py +++ b/libmodernize/main.py @@ -24,7 +24,6 @@ class LFPreservingRefactoringTool(StdoutRefactoringTool): def write_file(self, new_text, filename, old_text, encoding): # detect linefeeds lineends = {'\n':0, '\r\n':0, '\r':0} - lines = [] for line in open(filename, 'rb'): if line.endswith('\r\n'): lineends['\r\n'] += 1 @@ -32,7 +31,6 @@ def write_file(self, new_text, filename, old_text, encoding): lineends['\n'] += 1 elif line.endswith('\r'): lineends['\r'] += 1 - lines.append(line.rstrip('\r\n')) super(LFPreservingRefactoringTool, self).write_file( new_text, filename, old_text, encoding) # detect if line ends are consistent in source file @@ -41,8 +39,8 @@ def write_file(self, new_text, filename, old_text, encoding): newline = [x for x in lineends if lineends[x] != 0][0] if os.linesep != newline: with open(filename, 'wb') as f: - for line in lines: - f.write(line) + for line in new_text.splitlines(): + f.write(line + newline) self.log_debug('fixed %s linefeeds back to %s', filename, newline) From 3d6a5207d30f2712c89a862340ff852f51bb3716 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 19 Oct 2015 14:31:16 +0300 Subject: [PATCH 5/9] Convert \r\n to constants for Python 3 --- libmodernize/main.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/libmodernize/main.py b/libmodernize/main.py index 35f99f2..f992364 100644 --- a/libmodernize/main.py +++ b/libmodernize/main.py @@ -19,18 +19,29 @@ from libmodernize.fixes import lib2to3_fix_names, six_fix_names, opt_in_fix_names +PY3K = sys.version_info >= (3, 0) +if not PY3K: + LF = '\n' + CRLF = '\r\n' + CR = '\r' +else: + LF = bytes('\n', encoding='ascii') + CRLF = bytes('\r\n', encoding='ascii') + CR = bytes('\r', encoding='ascii') + + class LFPreservingRefactoringTool(StdoutRefactoringTool): """ https://github.com/python-modernize/python-modernize/issues/121 """ def write_file(self, new_text, filename, old_text, encoding): # detect linefeeds - lineends = {'\n':0, '\r\n':0, '\r':0} + lineends = {LF:0, CRLF:0, CR:0} for line in open(filename, 'rb'): - if line.endswith('\r\n'): - lineends['\r\n'] += 1 - elif line.endswith('\n'): - lineends['\n'] += 1 - elif line.endswith('\r'): - lineends['\r'] += 1 + if line.endswith(CRLF): + lineends[CRLF] += 1 + elif line.endswith(LF): + lineends[LF] += 1 + elif line.endswith(CR): + lineends[CR] += 1 super(LFPreservingRefactoringTool, self).write_file( new_text, filename, old_text, encoding) # detect if line ends are consistent in source file From b7c81aab860b9c9569acb9c82776c6c10bfe0890 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 19 Oct 2015 14:39:41 +0300 Subject: [PATCH 6/9] Reread source file as binary for Python 3 --- libmodernize/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libmodernize/main.py b/libmodernize/main.py index f992364..fe48eec 100644 --- a/libmodernize/main.py +++ b/libmodernize/main.py @@ -49,8 +49,13 @@ def write_file(self, new_text, filename, old_text, encoding): # detect if line ends are different from system-specific newline = [x for x in lineends if lineends[x] != 0][0] if os.linesep != newline: + # rereading new file is easier that writing new_text + # correct encoding in Python 2 and 3 compatible way + lines = [] + for line in open(filename, 'rb'): + lines.append(line.rstrip(CRLF)) with open(filename, 'wb') as f: - for line in new_text.splitlines(): + for line in lines: f.write(line + newline) self.log_debug('fixed %s linefeeds back to %s', filename, newline) From ff8f479e47c2d9a22e26c436a8a541270b422d43 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 19 Oct 2015 14:48:28 +0300 Subject: [PATCH 7/9] Prevent ResourceWarning's --- libmodernize/main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libmodernize/main.py b/libmodernize/main.py index fe48eec..02b88f0 100644 --- a/libmodernize/main.py +++ b/libmodernize/main.py @@ -34,14 +34,16 @@ class LFPreservingRefactoringTool(StdoutRefactoringTool): """ https://github.com/python-modernize/python-modernize/issues/121 """ def write_file(self, new_text, filename, old_text, encoding): # detect linefeeds + oldfile = open(filename, 'rb') lineends = {LF:0, CRLF:0, CR:0} - for line in open(filename, 'rb'): + for line in oldfile: if line.endswith(CRLF): lineends[CRLF] += 1 elif line.endswith(LF): lineends[LF] += 1 elif line.endswith(CR): lineends[CR] += 1 + oldfile.close() super(LFPreservingRefactoringTool, self).write_file( new_text, filename, old_text, encoding) # detect if line ends are consistent in source file @@ -52,8 +54,9 @@ def write_file(self, new_text, filename, old_text, encoding): # rereading new file is easier that writing new_text # correct encoding in Python 2 and 3 compatible way lines = [] - for line in open(filename, 'rb'): - lines.append(line.rstrip(CRLF)) + with open(filename, 'rb') as newfile: + for line in newfile: + lines.append(line.rstrip(CRLF)) with open(filename, 'wb') as f: for line in lines: f.write(line + newline) From e51e19fdf4e9228237a8c5049810ebf644f5699e Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 19 Oct 2015 16:18:11 +0300 Subject: [PATCH 8/9] Use bytes literals which are compatible back to Python 2.6 --- libmodernize/main.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/libmodernize/main.py b/libmodernize/main.py index 02b88f0..8224bda 100644 --- a/libmodernize/main.py +++ b/libmodernize/main.py @@ -19,16 +19,9 @@ from libmodernize.fixes import lib2to3_fix_names, six_fix_names, opt_in_fix_names -PY3K = sys.version_info >= (3, 0) -if not PY3K: - LF = '\n' - CRLF = '\r\n' - CR = '\r' -else: - LF = bytes('\n', encoding='ascii') - CRLF = bytes('\r\n', encoding='ascii') - CR = bytes('\r', encoding='ascii') - +LF = b'\n' +CRLF = b'\r\n' +CR = b'\r' class LFPreservingRefactoringTool(StdoutRefactoringTool): """ https://github.com/python-modernize/python-modernize/issues/121 """ From 4e45b5ac58b53e8458c2c91ee9f16ec36871bbef Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Thu, 22 Oct 2015 23:52:30 +0300 Subject: [PATCH 9/9] Fix indentation --- libmodernize/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmodernize/main.py b/libmodernize/main.py index 8224bda..d248f2a 100644 --- a/libmodernize/main.py +++ b/libmodernize/main.py @@ -166,7 +166,7 @@ def main(args=None): requested = default_fixes fixer_names = requested.difference(unwanted_fixes) rt = LFPreservingRefactoringTool(sorted(fixer_names), flags, sorted(explicit), - options.nobackups, not options.no_diffs) + options.nobackups, not options.no_diffs) # Refactor all files and directories passed as arguments if not rt.errors: