From b4a1f9b2b073a8485f94f7cf10d51a169306059d Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Sun, 5 Jan 2025 17:25:09 +0100 Subject: [PATCH 1/3] Make the docstring of untokenize() match the docs. The docstring previously claimed full roundtrip invariance, which is not the case. --- Lib/tokenize.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 7ece4e9b70d31b..1a60fd32a77ea4 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -318,16 +318,10 @@ def untokenize(iterable): with at least two elements, a token number and token value. If only two tokens are passed, the resulting output is poor. - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited input: - # Output bytes will tokenize back to the input - t1 = [tok[:2] for tok in tokenize(f.readline)] - newcode = untokenize(t1) - readline = BytesIO(newcode).readline - t2 = [tok[:2] for tok in tokenize(readline)] - assert t1 == t2 + The result is guaranteed to tokenize back to match the input so + that the conversion is lossless and round-trips are assured. + The guarantee applies only to the token type and token string as + the spacing between tokens (column positions) may change. """ ut = Untokenizer() out = ut.untokenize(iterable) From 373427ace4696e089e4cb32e4b4233327bb5ad6f Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Sun, 5 Jan 2025 18:41:42 +0100 Subject: [PATCH 2/3] Add news entry --- .../next/Library/2025-01-05-18-41-27.gh-issue-128519.DfI2M2.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-01-05-18-41-27.gh-issue-128519.DfI2M2.rst diff --git a/Misc/NEWS.d/next/Library/2025-01-05-18-41-27.gh-issue-128519.DfI2M2.rst b/Misc/NEWS.d/next/Library/2025-01-05-18-41-27.gh-issue-128519.DfI2M2.rst new file mode 100644 index 00000000000000..7447dc9474ef53 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-05-18-41-27.gh-issue-128519.DfI2M2.rst @@ -0,0 +1,2 @@ +Update the docstring of :func:`tokenize.untokenize` to match the +documentation. From 5467d3730b25556bfe0fc4f4133d062a6cb4477c Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Mon, 6 Jan 2025 09:17:23 +0100 Subject: [PATCH 3/3] Remove news entry --- .../next/Library/2025-01-05-18-41-27.gh-issue-128519.DfI2M2.rst | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2025-01-05-18-41-27.gh-issue-128519.DfI2M2.rst diff --git a/Misc/NEWS.d/next/Library/2025-01-05-18-41-27.gh-issue-128519.DfI2M2.rst b/Misc/NEWS.d/next/Library/2025-01-05-18-41-27.gh-issue-128519.DfI2M2.rst deleted file mode 100644 index 7447dc9474ef53..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-01-05-18-41-27.gh-issue-128519.DfI2M2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update the docstring of :func:`tokenize.untokenize` to match the -documentation.