diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py index 1bb1823b9f04..678ce8caec4e 100644 --- a/pythonFiles/normalizeForInterpreter.py +++ b/pythonFiles/normalizeForInterpreter.py @@ -77,12 +77,17 @@ def normalize_lines(source): """ lines = source.splitlines(False) # Find out if we have any trailing blank lines - has_blank_lines = len(lines[-1].strip()) == 0 or source.endswith(os.linesep) + if len(lines[-1].strip()) == 0 or source.endswith('\n'): + trailing_newline = '\n' + else: + trailing_newline = '' # Step 1: Remove empty lines. tokens = _tokenize(source) newlines_indexes_to_remove = (spos[0] for (toknum, tokval, spos, epos, line) in tokens - if len(line.strip()) == 0 and token.tok_name[toknum] == 'NL' and spos[0] == epos[0]) + if len(line.strip()) == 0 + and token.tok_name[toknum] == 'NL' + and spos[0] == epos[0]) for line_number in reversed(list(newlines_indexes_to_remove)): del lines[line_number-1] @@ -90,17 +95,17 @@ def normalize_lines(source): # Step 2: Add blank lines between each global statement block. # A consequtive single lines blocks of code will be treated as a single statement, # just to ensure we do not unnecessarily add too many blank lines. - source = os.linesep.join(lines) + source = '\n'.join(lines) tokens = _tokenize(source) dedent_indexes = (spos[0] for (toknum, tokval, spos, epos, line) in tokens if toknum == token.DEDENT and _indent_size(line) == 0) global_statement_ranges = _get_global_statement_blocks(source, lines) - - for line_number in filter(lambda x: x > 1, map(operator.itemgetter(0), reversed(global_statement_ranges))): + start_positions = map(operator.itemgetter(0), reversed(global_statement_ranges)) + for line_number in filter(lambda x: x > 1, start_positions): lines.insert(line_number-1, '') - sys.stdout.write(os.linesep.join(lines) + (os.linesep if has_blank_lines else '')) + sys.stdout.write('\n'.join(lines) + trailing_newline) sys.stdout.flush() @@ -108,7 +113,8 @@ def normalize_lines(source): contents = sys.argv[1] try: default_encoding = sys.getdefaultencoding() - contents = contents.encode(default_encoding, 'surrogateescape').decode(default_encoding, 'replace') + encoded_contents = contents.encode(default_encoding, 'surrogateescape') + contents = encoded_contents.decode(default_encoding, 'replace') except (UnicodeError, LookupError): pass if isinstance(contents, bytes): diff --git a/src/test/terminals/codeExecution/helper.test.ts b/src/test/terminals/codeExecution/helper.test.ts index 6c6ddeb9ca8f..56db9e5657cd 100644 --- a/src/test/terminals/codeExecution/helper.test.ts +++ b/src/test/terminals/codeExecution/helper.test.ts @@ -11,6 +11,7 @@ import * as TypeMoq from 'typemoq'; import { Range, Selection, TextDocument, TextEditor, TextLine, Uri } from 'vscode'; import { IApplicationShell, IDocumentManager } from '../../../client/common/application/types'; import { EXTENSION_ROOT_DIR, PYTHON_LANGUAGE } from '../../../client/common/constants'; +import '../../../client/common/extensions'; import { BufferDecoder } from '../../../client/common/process/decoder'; import { ProcessService } from '../../../client/common/process/proc'; import { IProcessService } from '../../../client/common/process/types'; @@ -62,6 +63,8 @@ suite('Terminal - Code Execution Helper', () => { return actualProcessService.exec.apply(actualProcessService, [file, args, options]); }); const normalizedZCode = await helper.normalizeLines(source); + // In case file has been saved with different line endings. + expectedSource = expectedSource.splitLines({ removeEmptyEntries: false, trim: false }).join(EOL); expect(normalizedZCode).to.be.equal(expectedSource); } test('Ensure blank lines are NOT removed when code is not indented (simple)', async () => {