Skip to content

Commit

Permalink
Relax restrictions about backslashes in pep701 f-strings
Browse files Browse the repository at this point in the history
  • Loading branch information
dflook committed Jan 11, 2024
1 parent 0a7cd60 commit 710f128
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 45 deletions.
4 changes: 2 additions & 2 deletions corpus_test/generate_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,9 +338,9 @@ def format_size_change_detail() -> str:
got_smaller_count = len(list(summary.compare_size_decrease(base_summary)))

if got_bigger_count > 0:
s += f', {got_bigger_count}:chart_with_upwards_trend:'
s += f', {got_bigger_count} :chart_with_upwards_trend:'
if got_smaller_count > 0:
s += f', {got_smaller_count}:chart_with_downwards_trend:'
s += f', {got_smaller_count} :chart_with_downwards_trend:'

s += ')'

Expand Down
6 changes: 3 additions & 3 deletions src/python_minifier/expression_printer.py
Original file line number Diff line number Diff line change
Expand Up @@ -736,11 +736,11 @@ def visit_JoinedStr(self, node):
import python_minifier.f_string

if sys.version_info < (3, 12):
quote_reuse = False
pep701 = False
else:
quote_reuse = True
pep701 = True

self.printer.fstring(str(python_minifier.f_string.OuterFString(node, quote_reuse=quote_reuse)))
self.printer.fstring(str(python_minifier.f_string.OuterFString(node, pep701=pep701)))

def visit_NamedExpr(self, node):
self._expression(node.target)
Expand Down
49 changes: 28 additions & 21 deletions src/python_minifier/f_string.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ class FString(object):
An F-string in the expression part of another f-string
"""

def __init__(self, node, allowed_quotes, quote_reuse):
def __init__(self, node, allowed_quotes, pep701):
assert isinstance(node, ast.JoinedStr)

self.node = node
self.allowed_quotes = allowed_quotes
self.quote_reuse = quote_reuse
self.pep701 = pep701

def is_correct_ast(self, code):
try:
Expand All @@ -54,7 +54,7 @@ def complete_debug_specifier(self, partial_specifier_candidates, value_node):
conversion_candidates = [x + conversion for x in partial_specifier_candidates]

if value_node.format_spec is not None:
conversion_candidates = [c + ':' + fs for c in conversion_candidates for fs in FormatSpec(value_node.format_spec, self.allowed_quotes, self.quote_reuse).candidates()]
conversion_candidates = [c + ':' + fs for c in conversion_candidates for fs in FormatSpec(value_node.format_spec, self.allowed_quotes, self.pep701).candidates()]

return [x + '}' for x in conversion_candidates]

Expand All @@ -66,7 +66,7 @@ def candidates(self):
debug_specifier_candidates = []
nested_allowed = copy.copy(self.allowed_quotes)

if not self.quote_reuse:
if not self.pep701:
nested_allowed.remove(quote)

for v in self.node.values:
Expand All @@ -90,7 +90,7 @@ def candidates(self):
try:
completed = self.complete_debug_specifier(debug_specifier_candidates, v)
candidates = [
x + y for x in candidates for y in FormattedValue(v, nested_allowed, self.quote_reuse).get_candidates()
x + y for x in candidates for y in FormattedValue(v, nested_allowed, self.pep701).get_candidates()
] + completed
debug_specifier_candidates = []
except Exception as e:
Expand All @@ -115,9 +115,9 @@ class OuterFString(FString):
OuterFString is free to use backslashes in the Str parts
"""

def __init__(self, node, quote_reuse=False):
def __init__(self, node, pep701=False):
assert isinstance(node, ast.JoinedStr)
super(OuterFString, self).__init__(node, ['"', "'", '"""', "'''"], quote_reuse=quote_reuse)
super(OuterFString, self).__init__(node, ['"', "'", '"""', "'''"], pep701=pep701)

def __str__(self):
if len(self.node.values) == 0:
Expand Down Expand Up @@ -155,13 +155,13 @@ class FormattedValue(ExpressionPrinter):
An F-String Expression Part
"""

def __init__(self, node, allowed_quotes, quote_reuse):
def __init__(self, node, allowed_quotes, pep701):
super(FormattedValue, self).__init__()

assert isinstance(node, ast.FormattedValue)
self.node = node
self.allowed_quotes = allowed_quotes
self.quote_reuse = quote_reuse
self.pep701 = pep701
self.candidates = ['']

def get_candidates(self):
Expand All @@ -182,7 +182,7 @@ def get_candidates(self):

if self.node.format_spec is not None:
self.printer.delimiter(':')
self._append(FormatSpec(self.node.format_spec, self.allowed_quotes, quote_reuse=self.quote_reuse).candidates())
self._append(FormatSpec(self.node.format_spec, self.allowed_quotes, pep701=self.pep701).candidates())

self.printer.delimiter('}')

Expand Down Expand Up @@ -211,7 +211,7 @@ def is_curly(self, node):
return False

def visit_Str(self, node):
self.printer.append(str(Str(node.s, self.allowed_quotes)), TokenTypes.NonNumberLiteral)
self.printer.append(str(Str(node.s, self.allowed_quotes, self.pep701)), TokenTypes.NonNumberLiteral)

def visit_Bytes(self, node):
self.printer.append(str(Bytes(node.s, self.allowed_quotes)), TokenTypes.NonNumberLiteral)
Expand All @@ -220,7 +220,7 @@ def visit_JoinedStr(self, node):
assert isinstance(node, ast.JoinedStr)
if self.printer.previous_token in [TokenTypes.Identifier, TokenTypes.Keyword, TokenTypes.SoftKeyword]:
self.printer.delimiter(' ')
self._append(FString(node, allowed_quotes=self.allowed_quotes, quote_reuse=self.quote_reuse).candidates())
self._append(FString(node, allowed_quotes=self.allowed_quotes, pep701=self.pep701).candidates())

def _finalize(self):
self.candidates = [x + str(self.printer) for x in self.candidates]
Expand All @@ -235,20 +235,21 @@ class Str(object):
"""
A Str node inside an f-string expression
May use any of the allowed quotes, no backslashes!
May use any of the allowed quotes. In Python <3.12, backslashes are not allowed.
"""

def __init__(self, s, allowed_quotes):
def __init__(self, s, allowed_quotes, pep701=False):
self._s = s
self.allowed_quotes = allowed_quotes
self.current_quote = None
self.pep701 = pep701

def _can_quote(self, c):
if self.current_quote is None:
return False

if (c == '\n' or c == '\r') and len(self.current_quote) == 1:
if (c == '\n' or c == '\r') and len(self.current_quote) == 1 and not self.pep701:
return False

if c == self.current_quote[0]:
Expand All @@ -258,7 +259,7 @@ def _can_quote(self, c):

def _get_quote(self, c):
for quote in self.allowed_quotes:
if c == '\n' or c == '\r':
if not self.pep701 and (c == '\n' or c == '\r'):
if len(quote) == 3:
return quote
elif c != quote:
Expand All @@ -279,7 +280,13 @@ def _literals(self):

if l == '':
l += self.current_quote
l += c

if c == '\n':
l += '\\n'
elif c == '\r':
l += '\\r'
else:
l += c

if l:
l += self.current_quote
Expand All @@ -292,7 +299,7 @@ def __str__(self):
if '\0' in self._s or '\\' in self._s:
raise ValueError('Impossible to represent a %r character in f-string expression part')

if '\n' in self._s or '\r' in self._s:
if not self.pep701 and ('\n' in self._s or '\r' in self._s):
if '"""' not in self.allowed_quotes and "'''" not in self.allowed_quotes:
raise ValueError(
'Impossible to represent newline character in f-string expression part without a long quote'
Expand Down Expand Up @@ -324,12 +331,12 @@ class FormatSpec(object):
"""

def __init__(self, node, allowed_quotes, quote_reuse):
def __init__(self, node, allowed_quotes, pep701):
assert isinstance(node, ast.JoinedStr)

self.node = node
self.allowed_quotes = allowed_quotes
self.quote_reuse = quote_reuse
self.pep701 = pep701

def candidates(self):

Expand All @@ -339,7 +346,7 @@ def candidates(self):
candidates = [x + self.str_for(v.s) for x in candidates]
elif isinstance(v, ast.FormattedValue):
candidates = [
x + y for x in candidates for y in FormattedValue(v, self.allowed_quotes, self.quote_reuse).get_candidates()
x + y for x in candidates for y in FormattedValue(v, self.allowed_quotes, self.pep701).get_candidates()
]
else:
raise RuntimeError('Unexpected JoinedStr value')
Expand Down
19 changes: 0 additions & 19 deletions test/test_empty_fstring.py

This file was deleted.

105 changes: 105 additions & 0 deletions test/test_fstring.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import ast
import sys

import pytest

from python_minifier import unparse
from python_minifier.ast_compare import compare_ast


@pytest.mark.parametrize('statement', [
'f"{1=!r:.4}"',
'f"{1=:.4}"',
'f"{1=!s:.4}"',
'f"{1=:.4}"',
'f"{1}"',
'f"{1=}"',
'f"{1=!s}"',
'f"{1=!a}"'
])
def test_fstring_statement(statement):
if sys.version_info < (3, 8):
pytest.skip('f-string debug specifier added in python 3.8')

assert unparse(ast.parse(statement)) == statement

def test_pep0701():
if sys.version_info < (3, 12):
pytest.skip('f-string syntax is bonkers before python 3.12')

statement = 'f"{f"{f"{f"{"hello"}"}"}"}"'
assert unparse(ast.parse(statement)) == statement

statement = 'f"This is the playlist: {", ".join([])}"'
assert unparse(ast.parse(statement)) == statement

statement = 'f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"'
assert unparse(ast.parse(statement)) == statement

statement = """
f"This is the playlist: {", ".join([
'Take me back to Eden', # My, my, those eyes like fire
'Alkaline', # Not acid nor alkaline
'Ascensionism' # Take to the broken skies at last
])}"
"""
assert unparse(ast.parse(statement)) == 'f"This is the playlist: {", ".join(["Take me back to Eden","Alkaline","Ascensionism"])}"'

statement = '''print(f"This is the playlist: {"\N{BLACK HEART SUIT}".join(songs)}")'''
assert unparse(ast.parse(statement)) == statement

statement = '''f"Magic wand: {bag["wand"]}"'''
assert unparse(ast.parse(statement)) == statement

statement = """
f'''A complex trick: {
bag['bag'] # recursive bags!
}'''
"""
assert unparse(ast.parse(statement)) == 'f"A complex trick: {bag["bag"]}"'

statement = '''f"These are the things: {", ".join(things)}"'''
assert unparse(ast.parse(statement)) == statement

statement = '''f"{source.removesuffix(".py")}.c: $(srcdir)/{source}"'''
assert unparse(ast.parse(statement)) == statement

statement = '''f"{f"{f"infinite"}"}"+' '+f"{f"nesting!!!"}"'''
assert unparse(ast.parse(statement)) == statement

statement = '''f"{"\\n".join(a)}"'''
assert unparse(ast.parse(statement)) == statement

statement = '''f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"'''
assert unparse(ast.parse(statement)) == statement

statement = '''f"{"":*^{1:{1}}}"'''
assert unparse(ast.parse(statement)) == statement

#statement = '''f"{"":*^{1:{1:{1}}}}"'''
#assert unparse(ast.parse(statement)) == statement
# SyntaxError: f-string: expressions nested too deeply

statement = '''f"___{
x
}___"'''
assert unparse(ast.parse(statement)) == '''f"___{x}___"'''

statement = '''f"___{(
x
)}___"'''
assert unparse(ast.parse(statement)) == '''f"___{x}___"'''

def test_fstring_empty_str():
if sys.version_info < (3, 6):
pytest.skip('f-string expressions not allowed in python < 3.6')

source = r'''
f"""\
{fg_br}"""
'''

print(source)
expected_ast = ast.parse(source)
actual_ast = unparse(expected_ast)
compare_ast(expected_ast, ast.parse(actual_ast))

0 comments on commit 710f128

Please sign in to comment.