From 78ddc5cfdf295e1fa10a8d349c53271f0b65a1e9 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 27 May 2020 02:17:34 +0300 Subject: [PATCH 1/3] Refactor error handling code in Parser/pegen/pegen.c Set p->error_indicator in various places, where it's needed, but it's not done. Also refactor `_PyPegen_expect_soft_keyword` to avoid a double call of `PyBytes_AsString` and a double check for the `NAME` type. --- Parser/pegen/pegen.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index ee30c2c0688f89..c054a24bfc850a 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -756,24 +756,15 @@ _PyPegen_expect_token(Parser *p, int type) expr_ty _PyPegen_expect_soft_keyword(Parser *p, const char *keyword) { - if (p->mark == p->fill) { - if (_PyPegen_fill_token(p) < 0) { - p->error_indicator = 1; - return NULL; - } - } - Token *t = p->tokens[p->mark]; - if (t->type != NAME) { - return NULL; - } - char* s = PyBytes_AsString(t->bytes); - if (!s) { + expr_ty res = _PyPegen_name_token(p); + if (!res) { return NULL; } - if (strcmp(s, keyword) != 0) { + const char *s = PyUnicode_AsUTF8(res->v.Name.id); + if (!s || strcmp(s, keyword) != 0) { + p->error_indicator = 1; return NULL; } - expr_ty res = _PyPegen_name_token(p); return res; } @@ -800,10 +791,12 @@ _PyPegen_name_token(Parser *p) } char* s = PyBytes_AsString(t->bytes); if (!s) { + p->error_indicator = 1; return NULL; } PyObject *id = _PyPegen_new_identifier(p, s); if (id == NULL) { + p->error_indicator = 1; return NULL; } return Name(id, Load, t->lineno, t->col_offset, t->end_lineno, t->end_col_offset, @@ -896,6 +889,7 @@ _PyPegen_number_token(Parser *p) char *num_raw = PyBytes_AsString(t->bytes); if (num_raw == NULL) { + p->error_indicator = 1; return NULL; } @@ -908,11 +902,13 @@ _PyPegen_number_token(Parser *p) PyObject *c = parsenumber(num_raw); if (c == NULL) { + p->error_indicator = 1; return NULL; } if (PyArena_AddPyObject(p->arena, c) < 0) { Py_DECREF(c); + p->error_indicator = 1; return NULL; } From c9daf582b30376399fd2225a448fa73dcc063816 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 27 May 2020 17:04:43 +0300 Subject: [PATCH 2/3] Don't set the error indicator when the rule just fails without error --- Parser/pegen/pegen.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index c054a24bfc850a..94be68981f5d6b 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -761,10 +761,13 @@ _PyPegen_expect_soft_keyword(Parser *p, const char *keyword) return NULL; } const char *s = PyUnicode_AsUTF8(res->v.Name.id); - if (!s || strcmp(s, keyword) != 0) { + if (!s) { p->error_indicator = 1; return NULL; } + if (strcmp(s, keyword) != 0) { + return NULL; + } return res; } From 4366cee6084e4e1ba6b2ad81b14160a087980229 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 27 May 2020 18:35:14 +0300 Subject: [PATCH 3/3] Revert changes to _PyPegen_expect_soft_keyword --- Parser/pegen/pegen.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index 94be68981f5d6b..bb22b12c3972bf 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -756,11 +756,17 @@ _PyPegen_expect_token(Parser *p, int type) expr_ty _PyPegen_expect_soft_keyword(Parser *p, const char *keyword) { - expr_ty res = _PyPegen_name_token(p); - if (!res) { + if (p->mark == p->fill) { + if (_PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + return NULL; + } + } + Token *t = p->tokens[p->mark]; + if (t->type != NAME) { return NULL; } - const char *s = PyUnicode_AsUTF8(res->v.Name.id); + char *s = PyBytes_AsString(t->bytes); if (!s) { p->error_indicator = 1; return NULL; @@ -768,7 +774,7 @@ _PyPegen_expect_soft_keyword(Parser *p, const char *keyword) if (strcmp(s, keyword) != 0) { return NULL; } - return res; + return _PyPegen_name_token(p); } Token *