fix tokenization of tqs with escaped newline

asottile · Oct 13, 2022 · 3810c9a · 3810c9a
1 parent e620ebc
commit 3810c9a
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 5 deletions.
diff --git a/reorder_python_imports.py b/reorder_python_imports.py
@@ -27,8 +27,8 @@
 COMMENT = r'#[^\r\n]*'
 NAME = r'\w+'
 PREFIX = r'[RrUu]?'
-DOUBLE_3 = r'"""[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
-SINGLE_3 = r"'''[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
+DOUBLE_3 = r'"""[^"\\]*(?:(?:\\.|\\\n|"(?!""))[^"\\]*)*"""'
+SINGLE_3 = r"'''[^'\\]*(?:(?:\\.|\\\n|'(?!''))[^'\\]*)*'''"
 DOUBLE_1 = r'"[^"\\]*(?:\\.[^"\\]*)*"'
 SINGLE_1 = r"'[^'\\]*(?:\\.[^'\\]*)*'"
 # END GENERATED

diff --git a/testing/generate-tokenize b/testing/generate-tokenize
@@ -19,12 +19,16 @@ def raw(s: str) -> str:
         return repr(s)
 
 
+def _escaped_newline(s: str) -> str:
+    return s.replace(r'\\.|', r'\\.|\\\n|')
+
+
 def main() -> int:
     # only want the prefixes that can be docstrings
     string_prefix = '[RrUu]?'
 
-    double_3 = f'"""{tokenize.Double3}'
-    single_3 = f"'''{tokenize.Single3}"
+    double_3 = f'"""{_escaped_newline(tokenize.Double3)}'
+    single_3 = f"'''{_escaped_newline(tokenize.Single3)}"
     double_1 = f'"{tokenize.Double}'
     single_1 = f"'{tokenize.Single}"
 

diff --git a/tests/reorder_python_imports_test.py b/tests/reorder_python_imports_test.py
@@ -26,7 +26,17 @@ def in_tmpdir(tmpdir):
 
 
 @pytest.mark.parametrize(
-    's', ("''", '""', 'r"hi"', 'u"hello"', '"""hello\nworld"""', "'''\n'''"),
+    's',
+    (
+        "''",
+        '""',
+        'r"hi"',
+        'u"hello"',
+        '"""hello\nworld"""',
+        "'''\n'''",
+        "'''\\\n'''",
+        "'''\\\r\n'''",
+    ),
 )
 def test_tokenize_can_match_strings(s):
     tp, pat = TOKENIZE[-1]