From 62d12da2e331854fa0160e13d9a1e3948f1381ad Mon Sep 17 00:00:00 2001 From: Sergii Khomenko Date: Sun, 14 Apr 2019 15:12:57 +0200 Subject: [PATCH 1/5] Add support for line splits --- tensorflow/tools/compatibility/ipynb.py | 40 +++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/tensorflow/tools/compatibility/ipynb.py b/tensorflow/tools/compatibility/ipynb.py index d37a1abde25030..d2656e1321c66c 100644 --- a/tensorflow/tools/compatibility/ipynb.py +++ b/tensorflow/tools/compatibility/ipynb.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================= -"""A module to support operation on ipynb files""" +"""A module to support operations on ipynb files""" from __future__ import absolute_import from __future__ import division @@ -62,8 +62,36 @@ def process_file(in_filename, out_filename, upgrader): return files_processed, report_text, errors +def skip_magic(code_line, magic_list): + """ + Checks if the cell has magic, that is not python based + + >>> skip_magic('!ls -laF') + True + """ + + for magic in magic_list: + if code_line.startswith(magic): + return True + + return False + + +def check_line_split(code_line): + r""" + Checks if line was splitted with `\` + >>> skip_magic("!gcloud ml-engine models create ${MODEL} \\\n") + True + """ + + if code_line.endswith('\\\n'): + return True + + return False + + def _get_code(input_file): - """Load the ipynb file and return a list of CodeLines.""" + """Loads the ipynb file and returns a list of CodeLines.""" raw_code = [] @@ -75,15 +103,17 @@ def _get_code(input_file): if is_python(cell): cell_lines = cell["source"] + is_line_split = False for line_idx, code_line in enumerate(cell_lines): # Sometimes, jupyter has more than python code # Idea is to comment these lines, for upgrade time - if code_line.startswith("%") or code_line.startswith("!") \ - or code_line.startswith("?"): + if skip_magic(code_line, ['%', '!', '?']) or is_line_split: # Found a special character, need to "encode" code_line = "###!!!" + code_line + is_line_split = check_line_split(code_line) + # Sometimes, people leave \n at the end of cell # in order to migrate only related things, and make the diff # the smallest -> here is another hack @@ -102,7 +132,7 @@ def _get_code(input_file): def _update_notebook(original_notebook, original_raw_lines, updated_code_lines): - """Update notebook, once migration is done.""" + """Updates notebook, once migration is done.""" new_notebook = copy.deepcopy(original_notebook) From 8fb20dd0b8206409fd46905aea673fbd2ca76282 Mon Sep 17 00:00:00 2001 From: Sergii Khomenko Date: Sun, 14 Apr 2019 20:13:54 +0200 Subject: [PATCH 2/5] Update doc test --- tensorflow/tools/compatibility/ipynb.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/tools/compatibility/ipynb.py b/tensorflow/tools/compatibility/ipynb.py index d2656e1321c66c..c6275ae0c8600c 100644 --- a/tensorflow/tools/compatibility/ipynb.py +++ b/tensorflow/tools/compatibility/ipynb.py @@ -64,9 +64,9 @@ def process_file(in_filename, out_filename, upgrader): def skip_magic(code_line, magic_list): """ - Checks if the cell has magic, that is not python based + Checks if the cell has magic, that is not python-based. - >>> skip_magic('!ls -laF') + >>> skip_magic('!ls -laF', ['%', '!', '?']) True """ @@ -79,7 +79,8 @@ def skip_magic(code_line, magic_list): def check_line_split(code_line): r""" - Checks if line was splitted with `\` + Checks if a line was splitted with `\`. + >>> skip_magic("!gcloud ml-engine models create ${MODEL} \\\n") True """ From fab6ba6e06cf95c51f628110765d16f232158f2e Mon Sep 17 00:00:00 2001 From: Sergii Khomenko Date: Mon, 15 Apr 2019 22:05:26 +0200 Subject: [PATCH 3/5] Comment line breaks only for magic lines --- tensorflow/tools/compatibility/ipynb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/tools/compatibility/ipynb.py b/tensorflow/tools/compatibility/ipynb.py index c6275ae0c8600c..e2c41097f46a8d 100644 --- a/tensorflow/tools/compatibility/ipynb.py +++ b/tensorflow/tools/compatibility/ipynb.py @@ -113,7 +113,8 @@ def _get_code(input_file): # Found a special character, need to "encode" code_line = "###!!!" + code_line - is_line_split = check_line_split(code_line) + # if this cell ends with `\` -> skip the next line + is_line_split = check_line_split(code_line) # Sometimes, people leave \n at the end of cell # in order to migrate only related things, and make the diff From f57940fc5478e23d15a94adcabf4621aaf9123c2 Mon Sep 17 00:00:00 2001 From: Sergii Khomenko Date: Mon, 15 Apr 2019 22:24:09 +0200 Subject: [PATCH 4/5] Continue line break only if one has started --- tensorflow/tools/compatibility/ipynb.py | 37 +++++++++++++------------ 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/tensorflow/tools/compatibility/ipynb.py b/tensorflow/tools/compatibility/ipynb.py index e2c41097f46a8d..1d6cbfdbe5780b 100644 --- a/tensorflow/tools/compatibility/ipynb.py +++ b/tensorflow/tools/compatibility/ipynb.py @@ -63,32 +63,32 @@ def process_file(in_filename, out_filename, upgrader): def skip_magic(code_line, magic_list): - """ - Checks if the cell has magic, that is not python-based. + """ + Checks if the cell has magic, that is not python-based. - >>> skip_magic('!ls -laF', ['%', '!', '?']) - True - """ + >>> skip_magic('!ls -laF', ['%', '!', '?']) + True + """ - for magic in magic_list: - if code_line.startswith(magic): - return True + for magic in magic_list: + if code_line.startswith(magic): + return True - return False + return False def check_line_split(code_line): - r""" - Checks if a line was splitted with `\`. + r""" + Checks if a line was splitted with `\`. - >>> skip_magic("!gcloud ml-engine models create ${MODEL} \\\n") - True - """ + >>> skip_magic("!gcloud ml-engine models create ${MODEL} \\\n") + True + """ - if code_line.endswith('\\\n'): - return True + if code_line.endswith('\\\n'): + return True - return False + return False def _get_code(input_file): @@ -116,6 +116,9 @@ def _get_code(input_file): # if this cell ends with `\` -> skip the next line is_line_split = check_line_split(code_line) + if is_line_split: + is_line_split = check_line_split(code_line) + # Sometimes, people leave \n at the end of cell # in order to migrate only related things, and make the diff # the smallest -> here is another hack From 850d3855648f279570667b2cf55e30da4be7caae Mon Sep 17 00:00:00 2001 From: Sergii Khomenko Date: Wed, 17 Apr 2019 12:29:16 +0200 Subject: [PATCH 5/5] Switch to regex; update styles --- tensorflow/tools/compatibility/ipynb.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tensorflow/tools/compatibility/ipynb.py b/tensorflow/tools/compatibility/ipynb.py index 1d6cbfdbe5780b..43674e2c75aaaf 100644 --- a/tensorflow/tools/compatibility/ipynb.py +++ b/tensorflow/tools/compatibility/ipynb.py @@ -21,6 +21,7 @@ import collections import copy import json +import re import shutil import tempfile @@ -63,8 +64,13 @@ def process_file(in_filename, out_filename, upgrader): def skip_magic(code_line, magic_list): - """ - Checks if the cell has magic, that is not python-based. + """Checks if the cell has magic, that is not Python-based. + + Args: + code_line: A line of Python code + magic_list: A list of jupyter "magic" exceptions + Returns: + If the line jupyter "magic" line, not Python line >>> skip_magic('!ls -laF', ['%', '!', '?']) True @@ -78,18 +84,18 @@ def skip_magic(code_line, magic_list): def check_line_split(code_line): - r""" - Checks if a line was splitted with `\`. + r"""Checks if a line was split with `\`. + + Args: + code_line: A line of Python code + Returns: + If the line was split with `\` >>> skip_magic("!gcloud ml-engine models create ${MODEL} \\\n") True """ - if code_line.endswith('\\\n'): - return True - - return False - + return re.search(r'\\\s*\n$', code_line) def _get_code(input_file): """Loads the ipynb file and returns a list of CodeLines."""