From 59c372a486bc18d04da01cbac627759291264034 Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Wed, 4 Oct 2023 07:21:01 +0800 Subject: [PATCH 1/4] prune empty constant in format specs --- Lib/test/test_fstring.py | 48 ++++++++++++++++++++++++++++++++++++++++ Parser/action_helpers.c | 40 ++++++++++++++++++++++++--------- 2 files changed, 78 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 4f05a149a901b2..dd8c2dd628ee13 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -514,6 +514,54 @@ def test_ast_fstring_empty_format_spec(self): self.assertEqual(type(format_spec), ast.JoinedStr) self.assertEqual(len(format_spec.values), 0) + def test_ast_fstring_format_spec(self): + expr = "f'{1:{name}}'" + + mod = ast.parse(expr) + self.assertEqual(type(mod), ast.Module) + self.assertEqual(len(mod.body), 1) + + fstring = mod.body[0].value + self.assertEqual(type(fstring), ast.JoinedStr) + self.assertEqual(len(fstring.values), 1) + + fv = fstring.values[0] + self.assertEqual(type(fv), ast.FormattedValue) + + format_spec = fv.format_spec + self.assertEqual(type(format_spec), ast.JoinedStr) + self.assertEqual(len(format_spec.values), 1) + + format_spec_value = format_spec.values[0] + self.assertEqual(type(format_spec_value), ast.FormattedValue) + self.assertEqual(format_spec_value.value.id, 'name') + + expr = "f'{1:{name1}{name2}}'" + + mod = ast.parse(expr) + self.assertEqual(type(mod), ast.Module) + self.assertEqual(len(mod.body), 1) + + fstring = mod.body[0].value + self.assertEqual(type(fstring), ast.JoinedStr) + self.assertEqual(len(fstring.values), 1) + + fv = fstring.values[0] + self.assertEqual(type(fv), ast.FormattedValue) + + format_spec = fv.format_spec + self.assertEqual(type(format_spec), ast.JoinedStr) + self.assertEqual(len(format_spec.values), 2) + + format_spec_value = format_spec.values[0] + self.assertEqual(type(format_spec_value), ast.FormattedValue) + self.assertEqual(format_spec_value.value.id, 'name1') + + format_spec_value = format_spec.values[1] + self.assertEqual(type(format_spec_value), ast.FormattedValue) + self.assertEqual(format_spec_value.value.id, 'name2') + + def test_docstring(self): def f(): f'''Not a docstring''' diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 36e0750220a30d..56f4a82d72d644 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -998,18 +998,38 @@ _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, in return NULL; } - // This is needed to keep compatibility with 3.11, where an empty format spec is parsed - // as an *empty* JoinedStr node, instead of having an empty constant in it. - if (asdl_seq_LEN(spec) == 1) { - expr_ty e = asdl_seq_GET(spec, 0); - if (e->kind == Constant_kind - && PyUnicode_Check(e->v.Constant.value) - && PyUnicode_GetLength(e->v.Constant.value) == 0) { - spec = _Py_asdl_expr_seq_new(0, arena); + // This is needed to keep compatibility with 3.11, where an empty format + // spec is parsed as an *empty* JoinedStr node, instead of having an empty + // constant in it. + Py_ssize_t n_items = asdl_seq_LEN(spec); + asdl_expr_seq *seq = _Py_asdl_expr_seq_new(n_items, p->arena); + if (seq == NULL) { + return NULL; + } + Py_ssize_t index = 0; + for (Py_ssize_t i = 0; i < n_items; i++) { + expr_ty item = asdl_seq_GET(spec, i); + if (item->kind == Constant_kind && + PyUnicode_CheckExact(item->v.Constant.value) && + PyUnicode_GET_LENGTH(item->v.Constant.value) == 0) { + continue; + } + asdl_seq_SET(seq, index++, item); + } + asdl_expr_seq *resized_specs; + if (index != n_items) { + resized_specs = _Py_asdl_expr_seq_new(index, p->arena); + if (resized_specs == NULL) { + return NULL; } + for (Py_ssize_t i = 0; i < index; i++) { + asdl_seq_SET(resized_specs, i, asdl_seq_GET(seq, i)); + } + } else { + resized_specs = seq; } - - expr_ty res = _PyAST_JoinedStr(spec, lineno, col_offset, end_lineno, end_col_offset, p->arena); + expr_ty res = _PyAST_JoinedStr(resized_specs, lineno, col_offset, + end_lineno, end_col_offset, p->arena); if (!res) { return NULL; } From 19d1301664d806ef958b778d16b721fa0fbf0825 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 23:26:20 +0000 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2023-10-03-23-26-18.gh-issue-110309.Y8nDOF.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-03-23-26-18.gh-issue-110309.Y8nDOF.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-03-23-26-18.gh-issue-110309.Y8nDOF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-03-23-26-18.gh-issue-110309.Y8nDOF.rst new file mode 100644 index 00000000000000..830428730391df --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-03-23-26-18.gh-issue-110309.Y8nDOF.rst @@ -0,0 +1 @@ +Remove unnecessary empty constant nodes in the ast of f-string specs. From 60b4666d7b43ba13a477d4d0fe6ddb1314f31775 Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Wed, 4 Oct 2023 07:39:57 +0800 Subject: [PATCH 3/4] count non empty nodes first --- Parser/action_helpers.c | 43 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 56f4a82d72d644..717c0f768fc4bc 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1002,34 +1002,35 @@ _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, in // spec is parsed as an *empty* JoinedStr node, instead of having an empty // constant in it. Py_ssize_t n_items = asdl_seq_LEN(spec); - asdl_expr_seq *seq = _Py_asdl_expr_seq_new(n_items, p->arena); - if (seq == NULL) { - return NULL; - } - Py_ssize_t index = 0; + Py_ssize_t non_empty_count = 0; for (Py_ssize_t i = 0; i < n_items; i++) { expr_ty item = asdl_seq_GET(spec, i); - if (item->kind == Constant_kind && - PyUnicode_CheckExact(item->v.Constant.value) && - PyUnicode_GET_LENGTH(item->v.Constant.value) == 0) { - continue; - } - asdl_seq_SET(seq, index++, item); - } - asdl_expr_seq *resized_specs; - if (index != n_items) { - resized_specs = _Py_asdl_expr_seq_new(index, p->arena); - if (resized_specs == NULL) { + non_empty_count += !(item->kind == Constant_kind && + PyUnicode_CheckExact(item->v.Constant.value) && + PyUnicode_GET_LENGTH(item->v.Constant.value) == 0); + } + asdl_expr_seq *resized_spec; + if (non_empty_count != n_items) { + resized_spec = _Py_asdl_expr_seq_new(non_empty_count, p->arena); + if (resized_spec == NULL) { return NULL; } - for (Py_ssize_t i = 0; i < index; i++) { - asdl_seq_SET(resized_specs, i, asdl_seq_GET(seq, i)); + Py_ssize_t j = 0; + for (Py_ssize_t i = 0; i < n_items; i++) { + expr_ty item = asdl_seq_GET(spec, i); + if (item->kind == Constant_kind && + PyUnicode_CheckExact(item->v.Constant.value) && + PyUnicode_GET_LENGTH(item->v.Constant.value) == 0) { + continue; + } + asdl_seq_SET(resized_spec, j++, item); } + assert(j == non_empty_count); } else { - resized_specs = seq; + resized_spec = spec; } - expr_ty res = _PyAST_JoinedStr(resized_specs, lineno, col_offset, - end_lineno, end_col_offset, p->arena); + expr_ty res = _PyAST_JoinedStr(resized_spec, lineno, col_offset, end_lineno, + end_col_offset, p->arena); if (!res) { return NULL; } From 19c68e2372870226446f7a493458d8e3768f2374 Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Wed, 4 Oct 2023 08:59:35 +0800 Subject: [PATCH 4/4] simplify things --- Parser/action_helpers.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 717c0f768fc4bc..b8713a329d4ef6 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1009,9 +1009,9 @@ _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, in PyUnicode_CheckExact(item->v.Constant.value) && PyUnicode_GET_LENGTH(item->v.Constant.value) == 0); } - asdl_expr_seq *resized_spec; if (non_empty_count != n_items) { - resized_spec = _Py_asdl_expr_seq_new(non_empty_count, p->arena); + asdl_expr_seq *resized_spec = + _Py_asdl_expr_seq_new(non_empty_count, p->arena); if (resized_spec == NULL) { return NULL; } @@ -1026,10 +1026,9 @@ _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, in asdl_seq_SET(resized_spec, j++, item); } assert(j == non_empty_count); - } else { - resized_spec = spec; + spec = resized_spec; } - expr_ty res = _PyAST_JoinedStr(resized_spec, lineno, col_offset, end_lineno, + expr_ty res = _PyAST_JoinedStr(spec, lineno, col_offset, end_lineno, end_col_offset, p->arena); if (!res) { return NULL;