Permalink
Browse files

Remove numeric underscore normalization (#696)

  • Loading branch information...
zsol authored and JelleZijlstra committed Feb 7, 2019
1 parent 36d3c51 commit 250ba7f04b300df284ba80cd4bb4122b45b41efb
Showing with 26 additions and 89 deletions.
  1. +4 −13 README.md
  2. +9 −54 black.py
  3. +0 −6 blackd.py
  4. +1 −1 tests/data/function.py
  5. +10 −10 tests/data/numeric_literals.py
  6. +2 −5 tests/test_black.py
@@ -87,9 +87,6 @@ Options:
piping source on standard input).
-S, --skip-string-normalization
Don't normalize string quotes or prefixes.
-N, --skip-numeric-underscore-normalization
Don't normalize underscores in numeric
literals.
--check Don't write the files back, just return the
status. Return code 0 means nothing would
change. Return code 1 means some files
@@ -395,14 +392,8 @@ an adoption helper, avoid using this for new projects.
*Black* standardizes most numeric literals to use lowercase letters for the
syntactic parts and uppercase letters for the digits themselves: `0xAB`
instead of `0XAB` and `1e10` instead of `1E10`. Python 2 long literals are
styled as `2L` instead of `2l` to avoid confusion between `l` and `1`. In
Python 3.6+, *Black* adds underscores to long numeric literals to aid
readability: `100000000` becomes `100_000_000`.
styled as `2L` instead of `2l` to avoid confusion between `l` and `1`.
For regions where numerals are grouped differently (like [India](https://en.wikipedia.org/wiki/Indian_numbering_system)
and [China](https://en.wikipedia.org/wiki/Chinese_numerals#Whole_numbers)),
the `-N` or `--skip-numeric-underscore-normalization` command line option
makes *Black* preserve underscores in numeric literals.
### Line breaks & binary operators
@@ -823,8 +814,6 @@ The headers controlling how code is formatted are:
- `X-Skip-String-Normalization`: corresponds to the `--skip-string-normalization`
command line flag. If present and its value is not the empty string, no string
normalization will be performed.
- `X-Skip-Numeric-Underscore-Normalization`: corresponds to the
`--skip-numeric-underscore-normalization` command line flag.
- `X-Fast-Or-Safe`: if set to `fast`, `blackd` will act as *Black* does when
passed the `--fast` command line flag.
- `X-Python-Variant`: if set to `pyi`, `blackd` will act as *Black* does when
@@ -950,7 +939,9 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).

## Change Log

### 18.11b0
### 19.2b0

* *Black* no longer normalizes numeric literals to include `_` separators.

* new option `--target-version` to control which Python versions
*Black*-formatted code should target
@@ -168,7 +168,6 @@ class Feature(Enum):
class FileMode:
target_versions: Set[TargetVersion] = Factory(set)
line_length: int = DEFAULT_LINE_LENGTH
numeric_underscore_normalization: bool = True
string_normalization: bool = True
is_pyi: bool = False

@@ -183,7 +182,6 @@ def get_cache_key(self) -> str:
parts = [
version_str,
str(self.line_length),
str(int(self.numeric_underscore_normalization)),
str(int(self.string_normalization)),
str(int(self.is_pyi)),
]
@@ -273,12 +271,6 @@ def read_pyproject_toml(
is_flag=True,
help="Don't normalize string quotes or prefixes.",
)
@click.option(
"-N",
"--skip-numeric-underscore-normalization",
is_flag=True,
help="Don't normalize underscores in numeric literals.",
)
@click.option(
"--check",
is_flag=True,
@@ -370,7 +362,6 @@ def main(
pyi: bool,
py36: bool,
skip_string_normalization: bool,
skip_numeric_underscore_normalization: bool,
quiet: bool,
verbose: bool,
include: str,
@@ -396,7 +387,6 @@ def main(
line_length=line_length,
is_pyi=pyi,
string_normalization=not skip_string_normalization,
numeric_underscore_normalization=not skip_numeric_underscore_normalization,
)
if config and verbose:
out(f"Using configuration from {config}.", bold=False, fg="blue")
@@ -686,8 +676,6 @@ def format_str(src_contents: str, *, mode: FileMode) -> FileContent:
or supports_feature(versions, Feature.UNICODE_LITERALS),
is_pyi=mode.is_pyi,
normalize_strings=mode.string_normalization,
allow_underscores=mode.numeric_underscore_normalization
and supports_feature(versions, Feature.NUMERIC_UNDERSCORES),
)
elt = EmptyLineTracker(is_pyi=mode.is_pyi)
empty_line = Line()
@@ -1492,7 +1480,6 @@ class LineGenerator(Visitor[Line]):
normalize_strings: bool = True
current_line: Line = Factory(Line)
remove_u_prefix: bool = False
allow_underscores: bool = False

def line(self, indent: int = 0) -> Iterator[Line]:
"""Generate a line.
@@ -1535,7 +1522,7 @@ def visit_default(self, node: LN) -> Iterator[Line]:
normalize_string_prefix(node, remove_u_prefix=self.remove_u_prefix)
normalize_string_quotes(node)
if node.type == token.NUMBER:
normalize_numeric_literal(node, self.allow_underscores)
normalize_numeric_literal(node)
if node.type not in WHITESPACE:
self.current_line.append(node)
yield from super().visit_default(node)
@@ -2674,11 +2661,11 @@ def normalize_string_quotes(leaf: Leaf) -> None:
leaf.value = f"{prefix}{new_quote}{new_body}{new_quote}"


def normalize_numeric_literal(leaf: Leaf, allow_underscores: bool) -> None:
def normalize_numeric_literal(leaf: Leaf) -> None:
"""Normalizes numeric (float, int, and complex) literals.
All letters used in the representation are normalized to lowercase (except
in Python 2 long literals), and long number literals are split using underscores.
in Python 2 long literals).
"""
text = leaf.value.lower()
if text.startswith(("0o", "0b")):
@@ -2696,59 +2683,27 @@ def normalize_numeric_literal(leaf: Leaf, allow_underscores: bool) -> None:
sign = "-"
elif after.startswith("+"):
after = after[1:]
before = format_float_or_int_string(before, allow_underscores)
after = format_int_string(after, allow_underscores)
before = format_float_or_int_string(before)
text = f"{before}e{sign}{after}"
elif text.endswith(("j", "l")):
number = text[:-1]
suffix = text[-1]
# Capitalize in "2L" because "l" looks too similar to "1".
if suffix == "l":
suffix = "L"
text = f"{format_float_or_int_string(number, allow_underscores)}{suffix}"
text = f"{format_float_or_int_string(number)}{suffix}"
else:
text = format_float_or_int_string(text, allow_underscores)
text = format_float_or_int_string(text)
leaf.value = text


def format_float_or_int_string(text: str, allow_underscores: bool) -> str:
def format_float_or_int_string(text: str) -> str:
"""Formats a float string like "1.0"."""
if "." not in text:
return format_int_string(text, allow_underscores)

before, after = text.split(".")
before = format_int_string(before, allow_underscores) if before else "0"
if after:
after = format_int_string(after, allow_underscores, count_from_end=False)
else:
after = "0"
return f"{before}.{after}"


def format_int_string(
text: str, allow_underscores: bool, count_from_end: bool = True
) -> str:
"""Normalizes underscores in a string to e.g. 1_000_000.
Input must be a string of digits and optional underscores.
If count_from_end is False, we add underscores after groups of three digits
counting from the beginning instead of the end of the strings. This is used
for the fractional part of float literals.
"""
if not allow_underscores:
return text

text = text.replace("_", "")
if len(text) <= 5:
# No underscores for numbers <= 5 digits long.
return text

if count_from_end:
# Avoid removing leading zeros, which are important if we're formatting
# part of a number like "0.001".
return format(int("1" + text), "3_")[1:].lstrip("_")
else:
return "_".join(text[i : i + 3] for i in range(0, len(text), 3))
before, after = text.split(".")
return f"{before or 0}.{after or 0}"


def normalize_invisible_parens(node: Node, parens_after: Set[str]) -> None:
@@ -17,15 +17,13 @@
LINE_LENGTH_HEADER = "X-Line-Length"
PYTHON_VARIANT_HEADER = "X-Python-Variant"
SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization"
SKIP_NUMERIC_UNDERSCORE_NORMALIZATION_HEADER = "X-Skip-Numeric-Underscore-Normalization"
FAST_OR_SAFE_HEADER = "X-Fast-Or-Safe"

BLACK_HEADERS = [
VERSION_HEADER,
LINE_LENGTH_HEADER,
PYTHON_VARIANT_HEADER,
SKIP_STRING_NORMALIZATION_HEADER,
SKIP_NUMERIC_UNDERSCORE_NORMALIZATION_HEADER,
FAST_OR_SAFE_HEADER,
]

@@ -95,9 +93,6 @@ def make_app() -> web.Application:
skip_string_normalization = bool(
request.headers.get(SKIP_STRING_NORMALIZATION_HEADER, False)
)
skip_numeric_underscore_normalization = bool(
request.headers.get(SKIP_NUMERIC_UNDERSCORE_NORMALIZATION_HEADER, False)
)
fast = False
if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast":
fast = True
@@ -106,7 +101,6 @@ def make_app() -> web.Application:
is_pyi=pyi,
line_length=line_length,
string_normalization=not skip_string_normalization,
numeric_underscore_normalization=not skip_numeric_underscore_normalization,
)
req_bytes = await request.content.read()
charset = request.charset if request.charset is not None else "utf8"
@@ -144,7 +144,7 @@ def function_signature_stress_test(


def spaces(a=1, b=(), c=[], d={}, e=True, f=-1, g=1 if False else 2, h="", i=r""):
offset = attr.ib(default=attr.Factory(lambda: _r.uniform(10000, 200_000)))
offset = attr.ib(default=attr.Factory(lambda: _r.uniform(10000, 200000)))
assert task._cancel_stack[: len(old_stack)] == old_stack


@@ -6,7 +6,7 @@
x = 1.
x = 1E+1
x = 1E-1
x = 1.00000001
x = 1.000_000_01
x = 123456789.123456789
x = 123456789.123456789E123456789
x = 123456789E123456789
@@ -24,21 +24,21 @@

#!/usr/bin/env python3.6

x = 123_456_789
x = 123_456
x = 123456789
x = 123456
x = 0.1
x = 1.0
x = 1e1
x = 1e-1
x = 1.000_000_01
x = 123_456_789.123_456_789
x = 123_456_789.123_456_789e123_456_789
x = 123_456_789e123_456_789
x = 123_456_789j
x = 123_456_789.123_456_789j
x = 123456789.123456789
x = 123456789.123456789e123456789
x = 123456789e123456789
x = 123456789j
x = 123456789.123456789j
x = 0xB1ACC
x = 0b1011
x = 0o777
x = 0.000_000_006
x = 0.000000006
x = 10000
x = 133_333
x = 133333
@@ -437,9 +437,7 @@ def test_numeric_literals(self) -> None:
@patch("black.dump_to_file", dump_to_stderr)
def test_numeric_literals_ignoring_underscores(self) -> None:
source, expected = read_data("numeric_literals_skip_underscores")
mode = black.FileMode(
numeric_underscore_normalization=False, target_versions=black.PY36_VERSIONS
)
mode = black.FileMode(target_versions=black.PY36_VERSIONS)
actual = fs(source, mode=mode)
self.assertFormatEqual(expected, actual)
black.assert_equivalent(source, actual)
@@ -828,8 +826,7 @@ def test_get_features_used(self) -> None:
)
node = black.lib2to3_parse(expected)
self.assertEqual(
black.get_features_used(node),
{Feature.TRAILING_COMMA, Feature.F_STRINGS, Feature.NUMERIC_UNDERSCORES},
black.get_features_used(node), {Feature.TRAILING_COMMA, Feature.F_STRINGS}
)
source, expected = read_data("expression")
node = black.lib2to3_parse(source)

0 comments on commit 250ba7f

Please sign in to comment.