From 6c708474ee239574695539edde088cd7c7cdfeb6 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Mon, 1 Sep 2025 16:09:36 +0800 Subject: [PATCH 1/9] fix: xx.keyword that keyword should not be highlight Signed-off-by: yihong0618 --- Lib/_pyrepl/utils.py | 6 ++- Lib/test/test_pyrepl/test_utils.py | 37 ++++++++++++++++++- ...-09-01-16-09-02.gh-issue-138318.t-WEN5.rst | 1 + 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index fd788c8429e15b2..d7ffbfc325c95d4 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -196,7 +196,8 @@ def gen_colors_from_token_stream( is_def_name = False span = Span.from_token(token, line_lengths) yield ColorSpan(span, "definition") - elif keyword.iskeyword(token.string): + elif (keyword.iskeyword(token.string) + and not (prev_token and prev_token.type == T.OP and prev_token.string == ".")): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "keyword") if token.string in IDENTIFIERS_AFTER: @@ -208,7 +209,8 @@ def gen_colors_from_token_stream( ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "soft_keyword") - elif token.string in BUILTINS: + elif (token.string in BUILTINS + and not (prev_token and prev_token.type == T.OP and prev_token.string == ".")): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "builtin") diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py index 8ce1e5371386f07..61520eb486302fa 100644 --- a/Lib/test/test_pyrepl/test_utils.py +++ b/Lib/test/test_pyrepl/test_utils.py @@ -1,6 +1,6 @@ from unittest import TestCase -from _pyrepl.utils import str_width, wlen, prev_next_window +from _pyrepl.utils import str_width, wlen, prev_next_window, gen_colors class TestUtils(TestCase): @@ -60,3 +60,38 @@ def gen_raise(): self.assertEqual(next(pnw), (3, 4, None)) with self.assertRaises(ZeroDivisionError): next(pnw) + + def test_gen_colors_keyword_highlighting(self): + no_highlight_cases = [ + ("a.set", [(".", "op")]), # 'set' should not be highlighted after '.' + ("a.def", [(".", "op")]), # 'def' should not be highlighted after '.' + ("obj.class", [(".", "op")]), # 'class' should not be highlighted after '.' + ("obj.list", [(".", "op")]), # 'list' should not be highlighted after '.' + ("obj.match", [(".", "op")]), # 'match' should not be highlighted after '.' + ] + for code, expected_highlights in no_highlight_cases: + with self.subTest(code=code): + colors = list(gen_colors(code)) + # Extract (text, tag) pairs for comparison + actual_highlights = [] + for color in colors: + span_text = code[color.span.start:color.span.end + 1] + actual_highlights.append((span_text, color.tag)) + self.assertEqual(actual_highlights, expected_highlights, + f"In '{code}', expected {expected_highlights}, got {actual_highlights}") + highlight_cases = [ + ("set", [("set", "builtin")]), + ("list", [("list", "builtin")]), + ("def func():", [("def", "keyword"), ("func", "definition"), ("(", "op"), (")", "op"), (":", "op")]), + ("class A:", [("class", "keyword"), ("A", "definition"), (":", "op")]), + ] + for code, expected_highlights in highlight_cases: + with self.subTest(code=code): + colors = list(gen_colors(code)) + # Extract (text, tag) pairs for comparison + actual_highlights = [] + for color in colors: + span_text = code[color.span.start:color.span.end + 1] + actual_highlights.append((span_text, color.tag)) + self.assertEqual(actual_highlights, expected_highlights, + f"In '{code}', expected {expected_highlights}, got {actual_highlights}") diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst new file mode 100644 index 000000000000000..e73ac45ffd66088 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst @@ -0,0 +1 @@ +fix: xxx.keyword the keyword should not highlight in new repl From 5377f66fb494ef399fedc2b6b2138b62c892f4c3 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Tue, 2 Sep 2025 10:47:49 +0800 Subject: [PATCH 2/9] fix: only the buildin words Signed-off-by: yihong0618 --- Lib/_pyrepl/utils.py | 3 +-- Lib/test/test_pyrepl/test_utils.py | 10 +++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index d7ffbfc325c95d4..f2b1ba90a9bde4d 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -196,8 +196,7 @@ def gen_colors_from_token_stream( is_def_name = False span = Span.from_token(token, line_lengths) yield ColorSpan(span, "definition") - elif (keyword.iskeyword(token.string) - and not (prev_token and prev_token.type == T.OP and prev_token.string == ".")): + elif keyword.iskeyword(token.string): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "keyword") if token.string in IDENTIFIERS_AFTER: diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py index 61520eb486302fa..23a45bc5bf1dac2 100644 --- a/Lib/test/test_pyrepl/test_utils.py +++ b/Lib/test/test_pyrepl/test_utils.py @@ -63,11 +63,9 @@ def gen_raise(): def test_gen_colors_keyword_highlighting(self): no_highlight_cases = [ - ("a.set", [(".", "op")]), # 'set' should not be highlighted after '.' - ("a.def", [(".", "op")]), # 'def' should not be highlighted after '.' - ("obj.class", [(".", "op")]), # 'class' should not be highlighted after '.' - ("obj.list", [(".", "op")]), # 'list' should not be highlighted after '.' - ("obj.match", [(".", "op")]), # 'match' should not be highlighted after '.' + ("a.set", [(".", "op")]), + ("obj.list", [(".", "op")]), + ("obj.match", [(".", "op")]), ] for code, expected_highlights in no_highlight_cases: with self.subTest(code=code): @@ -82,8 +80,6 @@ def test_gen_colors_keyword_highlighting(self): highlight_cases = [ ("set", [("set", "builtin")]), ("list", [("list", "builtin")]), - ("def func():", [("def", "keyword"), ("func", "definition"), ("(", "op"), (")", "op"), (":", "op")]), - ("class A:", [("class", "keyword"), ("A", "definition"), (":", "op")]), ] for code, expected_highlights in highlight_cases: with self.subTest(code=code): From 72574f50e4b538bfa7828f2e7aa6be50c8f12f6a Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Tue, 2 Sep 2025 11:18:54 +0800 Subject: [PATCH 3/9] fix: fix the news info Signed-off-by: yihong0618 --- .../2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst index e73ac45ffd66088..09018b95c13a5e0 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst @@ -1 +1 @@ -fix: xxx.keyword the keyword should not highlight in new repl +fix: xxx.builtins the builtins keyword should not highlight in new repl From a625cecd79611fc766bb92cec0d5fdea7c02b384 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Tue, 2 Sep 2025 17:20:38 +0800 Subject: [PATCH 4/9] fix: address comments Signed-off-by: yihong0618 --- Lib/_pyrepl/utils.py | 6 ++++-- Lib/test/test_pyrepl/test_utils.py | 21 +++++-------------- ...-09-01-16-09-02.gh-issue-138318.t-WEN5.rst | 4 +++- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index f2b1ba90a9bde4d..35e385f170ca438 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -208,8 +208,10 @@ def gen_colors_from_token_stream( ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "soft_keyword") - elif (token.string in BUILTINS - and not (prev_token and prev_token.type == T.OP and prev_token.string == ".")): + elif ( + token.string in BUILTINS + and not (prev_token and prev_token.type == T.OP + and prev_token.string == ".")): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "builtin") diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py index 23a45bc5bf1dac2..5d2a496d8d8ab5d 100644 --- a/Lib/test/test_pyrepl/test_utils.py +++ b/Lib/test/test_pyrepl/test_utils.py @@ -62,26 +62,16 @@ def gen_raise(): next(pnw) def test_gen_colors_keyword_highlighting(self): - no_highlight_cases = [ + cases = [ + # no highlights ("a.set", [(".", "op")]), ("obj.list", [(".", "op")]), ("obj.match", [(".", "op")]), - ] - for code, expected_highlights in no_highlight_cases: - with self.subTest(code=code): - colors = list(gen_colors(code)) - # Extract (text, tag) pairs for comparison - actual_highlights = [] - for color in colors: - span_text = code[color.span.start:color.span.end + 1] - actual_highlights.append((span_text, color.tag)) - self.assertEqual(actual_highlights, expected_highlights, - f"In '{code}', expected {expected_highlights}, got {actual_highlights}") - highlight_cases = [ + # highlights ("set", [("set", "builtin")]), ("list", [("list", "builtin")]), ] - for code, expected_highlights in highlight_cases: + for code, expected_highlights in cases: with self.subTest(code=code): colors = list(gen_colors(code)) # Extract (text, tag) pairs for comparison @@ -89,5 +79,4 @@ def test_gen_colors_keyword_highlighting(self): for color in colors: span_text = code[color.span.start:color.span.end + 1] actual_highlights.append((span_text, color.tag)) - self.assertEqual(actual_highlights, expected_highlights, - f"In '{code}', expected {expected_highlights}, got {actual_highlights}") + self.assertEqual(actual_highlights, expected_highlights) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst index 09018b95c13a5e0..d9d433b9ee9d659 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst @@ -1 +1,3 @@ -fix: xxx.builtins the builtins keyword should not highlight in new repl +The default REPL now avoids highlighting built-in names (for instance :class:`set` + or :func:`format`) when they are used as attribute names (for instance in ``value.set`` + or ``text.format``). From e89085fb46f422cd2814f2d8578e3ef0c3217e98 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Tue, 2 Sep 2025 19:36:58 +0800 Subject: [PATCH 5/9] fix: follow up comments Signed-off-by: yihong0618 --- Lib/_pyrepl/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 35e385f170ca438..3883787f96b6e4b 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -210,8 +210,7 @@ def gen_colors_from_token_stream( yield ColorSpan(span, "soft_keyword") elif ( token.string in BUILTINS - and not (prev_token and prev_token.type == T.OP - and prev_token.string == ".")): + and not (prev_token and prev_token.exact_type == T.DOT)): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "builtin") From 8e81baff1bd0cabd66dcb92d17121ee914e0d5f6 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Tue, 2 Sep 2025 19:41:34 +0800 Subject: [PATCH 6/9] fix: follow up comments again Signed-off-by: yihong0618 --- Lib/_pyrepl/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 3883787f96b6e4b..c5d006afa7731fa 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -210,7 +210,8 @@ def gen_colors_from_token_stream( yield ColorSpan(span, "soft_keyword") elif ( token.string in BUILTINS - and not (prev_token and prev_token.exact_type == T.DOT)): + and not (prev_token and prev_token.exact_type == T.DOT) + ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "builtin") From 2673564ed6664369117da89bf1fe985179501f60 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Wed, 3 Sep 2025 17:49:23 +0800 Subject: [PATCH 7/9] fix: more tests and address comments Signed-off-by: yihong0618 --- Lib/test/test_pyrepl/test_utils.py | 1 + .../2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py index 5d2a496d8d8ab5d..1389fc37da78baf 100644 --- a/Lib/test/test_pyrepl/test_utils.py +++ b/Lib/test/test_pyrepl/test_utils.py @@ -67,6 +67,7 @@ def test_gen_colors_keyword_highlighting(self): ("a.set", [(".", "op")]), ("obj.list", [(".", "op")]), ("obj.match", [(".", "op")]), + ("b. \\\n format", [(".", "op")]), # highlights ("set", [("set", "builtin")]), ("list", [("list", "builtin")]), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst index d9d433b9ee9d659..2bbd164a38e014c 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst @@ -1,3 +1,4 @@ The default REPL now avoids highlighting built-in names (for instance :class:`set` - or :func:`format`) when they are used as attribute names (for instance in ``value.set`` - or ``text.format``). +or :func:`format`) when they are used as attribute names (for instance in ``value.set`` +or ``text.format``). + From 31fe0100a2a2165b0301a7057630930cc410f458 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Wed, 3 Sep 2025 19:41:29 +0800 Subject: [PATCH 8/9] fix: drop empty line in news MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: yihong0618 Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst index 2bbd164a38e014c..ce9456ddd109541 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-01-16-09-02.gh-issue-138318.t-WEN5.rst @@ -1,4 +1,3 @@ The default REPL now avoids highlighting built-in names (for instance :class:`set` or :func:`format`) when they are used as attribute names (for instance in ``value.set`` or ``text.format``). - From 769530345d8e012aac8027e0c86853898652118f Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Sat, 6 Sep 2025 21:36:16 +0800 Subject: [PATCH 9/9] fix: add more highlights case Signed-off-by: yihong0618 --- Lib/test/test_pyrepl/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py index 1389fc37da78baf..05a4f3290598350 100644 --- a/Lib/test/test_pyrepl/test_utils.py +++ b/Lib/test/test_pyrepl/test_utils.py @@ -71,6 +71,7 @@ def test_gen_colors_keyword_highlighting(self): # highlights ("set", [("set", "builtin")]), ("list", [("list", "builtin")]), + (" \n dict", [("dict", "builtin")]), ] for code, expected_highlights in cases: with self.subTest(code=code):