From 39b83e3826d406bd830822c8feff863d64c47bec Mon Sep 17 00:00:00 2001 From: 10maurycy10 <45378255+10maurycy10@users.noreply.github.com> Date: Tue, 20 Apr 2021 09:34:07 -0800 Subject: [PATCH 1/3] Minor clean up. --- Parser/tokenizer.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index ad32293d70b785..04a91cb828be56 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -18,17 +18,18 @@ /* Alternate tab spacing */ #define ALTTABSIZE 1 +/* Helper macros for identifiers */ #define is_potential_identifier_start(c) (\ (c >= 'a' && c <= 'z')\ || (c >= 'A' && c <= 'Z')\ - || c == '_'\ + || (c == '_')\ || (c >= 128)) #define is_potential_identifier_char(c) (\ (c >= 'a' && c <= 'z')\ || (c >= 'A' && c <= 'Z')\ || (c >= '0' && c <= '9')\ - || c == '_'\ + || (c == '_')\ || (c >= 128)) From 28473e38f2f0dde3deeab396e8a12eafb9d87eac Mon Sep 17 00:00:00 2001 From: 10maurycy10 <10maurycy10@gmail.com> Date: Tue, 20 Apr 2021 10:53:42 -0800 Subject: [PATCH 2/3] more clean up --- Parser/tokenizer.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 04a91cb828be56..92c885f9db248b 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -23,14 +23,16 @@ (c >= 'a' && c <= 'z')\ || (c >= 'A' && c <= 'Z')\ || (c == '_')\ - || (c >= 128)) + || (c >= 128) + ) #define is_potential_identifier_char(c) (\ (c >= 'a' && c <= 'z')\ || (c >= 'A' && c <= 'Z')\ || (c >= '0' && c <= '9')\ || (c == '_')\ - || (c >= 128)) + || (c >= 128) + ) /* Don't ever change this -- it would break the portability of Python code */ From 814d953bb2841d436106cd7f802b444499c1cafc Mon Sep 17 00:00:00 2001 From: 10maurycy10 <10maurycy10@gmail.com> Date: Tue, 20 Apr 2021 10:56:23 -0800 Subject: [PATCH 3/3] Update tokenizer.c --- Parser/tokenizer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 92c885f9db248b..7fa923fca4ae44 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -23,7 +23,7 @@ (c >= 'a' && c <= 'z')\ || (c >= 'A' && c <= 'Z')\ || (c == '_')\ - || (c >= 128) + || (c >= 128)\ ) #define is_potential_identifier_char(c) (\ @@ -31,7 +31,7 @@ || (c >= 'A' && c <= 'Z')\ || (c >= '0' && c <= '9')\ || (c == '_')\ - || (c >= 128) + || (c >= 128)\ )