From c2843c906cc0cc48233d65baa3599bcab4ad08c6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 20 Mar 2019 00:28:00 +0000 Subject: [PATCH 1/5] bpo-36143: Regenerate Lib/keyword.py from the Grammar and Tokens file using pgen --- Lib/keyword.py | 59 ++------- Lib/test/test_keyword.py | 119 ++---------------- Makefile.pre.in | 11 +- .../2019-03-20-00-37-24.bpo-12456.fnKoKo.rst | 2 + Parser/pgen/keywordgen.py | 62 +++++++++ 5 files changed, 92 insertions(+), 161 deletions(-) mode change 100755 => 100644 Lib/keyword.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-03-20-00-37-24.bpo-12456.fnKoKo.rst create mode 100644 Parser/pgen/keywordgen.py diff --git a/Lib/keyword.py b/Lib/keyword.py old mode 100755 new mode 100644 index 150c2bc46d229c..605d7504abad54 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -1,25 +1,30 @@ #! /usr/bin/env python3 -"""Keywords (from "graminit.c") +"""Keywords (from "Grammar/Grammar") This file is automatically generated; please don't muck it up! To update the symbols in this file, 'cd' to the top directory of -the python source tree after building the interpreter and run: +the python source tree and run: - ./python Lib/keyword.py + python -m Parser.pgen.keywordgen ./Grammar/Grammar \ + ./Grammar/Tokens \ + ./Lib/keyword.py + +Alternatively, you can run 'make regen-keyword'. """ __all__ = ["iskeyword", "kwlist"] kwlist = [ -#--start keywords-- 'False', 'None', 'True', 'and', 'as', 'assert', + 'async', + 'await', 'break', 'class', 'continue', @@ -47,52 +52,6 @@ 'while', 'with', 'yield', -#--end keywords-- ] -kwlist.append('async') -kwlist.append('await') -kwlist.sort() - iskeyword = frozenset(kwlist).__contains__ - -def main(): - import sys, re - - args = sys.argv[1:] - iptfile = args and args[0] or "Python/graminit.c" - if len(args) > 1: optfile = args[1] - else: optfile = "Lib/keyword.py" - - # load the output skeleton from the target, taking care to preserve its - # newline convention. - with open(optfile, newline='') as fp: - format = fp.readlines() - nl = format[0][len(format[0].strip()):] if format else '\n' - - # scan the source file for keywords - with open(iptfile) as fp: - strprog = re.compile('"([^"]+)"') - lines = [] - for line in fp: - if '{1, "' in line: - match = strprog.search(line) - if match: - lines.append(" '" + match.group(1) + "'," + nl) - lines.sort() - - # insert the lines of keywords into the skeleton - try: - start = format.index("#--start keywords--" + nl) + 1 - end = format.index("#--end keywords--" + nl) - format[start:end] = lines - except ValueError: - sys.stderr.write("target does not contain format markers\n") - sys.exit(1) - - # write the output file - with open(optfile, 'w', newline='') as fp: - fp.writelines(format) - -if __name__ == "__main__": - main() diff --git a/Lib/test/test_keyword.py b/Lib/test/test_keyword.py index af99f52c630d3c..3e2a8b3fb7f4c3 100644 --- a/Lib/test/test_keyword.py +++ b/Lib/test/test_keyword.py @@ -1,20 +1,5 @@ import keyword import unittest -from test import support -import filecmp -import os -import sys -import subprocess -import shutil -import textwrap - -KEYWORD_FILE = support.findfile('keyword.py') -GRAMMAR_FILE = os.path.join(os.path.split(__file__)[0], - '..', '..', 'Python', 'graminit.c') -TEST_PY_FILE = 'keyword_test.py' -GRAMMAR_TEST_FILE = 'graminit_test.c' -PY_FILE_WITHOUT_KEYWORDS = 'minimal_keyword.py' -NONEXISTENT_FILE = 'not_here.txt' class Test_iskeyword(unittest.TestCase): @@ -35,103 +20,17 @@ def test_changing_the_kwlist_does_not_affect_iskeyword(self): keyword.kwlist = ['its', 'all', 'eggs', 'beans', 'and', 'a', 'slice'] self.assertFalse(keyword.iskeyword('eggs')) + def test_all_keywords_fail_to_be_used_as_names(self): + for key in keyword.kwlist: + with self.assertRaises(SyntaxError): + exec(f"{key} = 42") -class TestKeywordGeneration(unittest.TestCase): - - def _copy_file_without_generated_keywords(self, source_file, dest_file): - with open(source_file, 'rb') as fp: - lines = fp.readlines() - nl = lines[0][len(lines[0].strip()):] - with open(dest_file, 'wb') as fp: - fp.writelines(lines[:lines.index(b"#--start keywords--" + nl) + 1]) - fp.writelines(lines[lines.index(b"#--end keywords--" + nl):]) - - def _generate_keywords(self, grammar_file, target_keyword_py_file): - proc = subprocess.Popen([sys.executable, - KEYWORD_FILE, - grammar_file, - target_keyword_py_file], stderr=subprocess.PIPE) - stderr = proc.communicate()[1] - return proc.returncode, stderr - - @unittest.skipIf(not os.path.exists(GRAMMAR_FILE), - 'test only works from source build directory') - def test_real_grammar_and_keyword_file(self): - self._copy_file_without_generated_keywords(KEYWORD_FILE, TEST_PY_FILE) - self.addCleanup(support.unlink, TEST_PY_FILE) - self.assertFalse(filecmp.cmp(KEYWORD_FILE, TEST_PY_FILE)) - self.assertEqual((0, b''), self._generate_keywords(GRAMMAR_FILE, - TEST_PY_FILE)) - self.assertTrue(filecmp.cmp(KEYWORD_FILE, TEST_PY_FILE)) - - def test_grammar(self): - self._copy_file_without_generated_keywords(KEYWORD_FILE, TEST_PY_FILE) - self.addCleanup(support.unlink, TEST_PY_FILE) - with open(GRAMMAR_TEST_FILE, 'w') as fp: - # Some of these are probably implementation accidents. - fp.writelines(textwrap.dedent("""\ - {2, 1}, - {11, "encoding_decl", 0, 2, states_79, - "\000\000\040\000\000\000\000\000\000\000\000\000" - "\000\000\000\000\000\000\000\000\000"}, - {1, "jello"}, - {326, 0}, - {1, "turnip"}, - \t{1, "This one is tab indented" - {278, 0}, - {1, "crazy but legal" - "also legal" {1, " - {1, "continue"}, - {1, "lemon"}, - {1, "tomato"}, - {1, "wigii"}, - {1, 'no good'} - {283, 0}, - {1, "too many spaces"}""")) - self.addCleanup(support.unlink, GRAMMAR_TEST_FILE) - self._generate_keywords(GRAMMAR_TEST_FILE, TEST_PY_FILE) - expected = [ - " 'This one is tab indented',", - " 'also legal',", - " 'continue',", - " 'crazy but legal',", - " 'jello',", - " 'lemon',", - " 'tomato',", - " 'turnip',", - " 'wigii',", - ] - with open(TEST_PY_FILE) as fp: - lines = fp.read().splitlines() - start = lines.index("#--start keywords--") + 1 - end = lines.index("#--end keywords--") - actual = lines[start:end] - self.assertEqual(actual, expected) - - def test_empty_grammar_results_in_no_keywords(self): - self._copy_file_without_generated_keywords(KEYWORD_FILE, - PY_FILE_WITHOUT_KEYWORDS) - self.addCleanup(support.unlink, PY_FILE_WITHOUT_KEYWORDS) - shutil.copyfile(KEYWORD_FILE, TEST_PY_FILE) - self.addCleanup(support.unlink, TEST_PY_FILE) - self.assertEqual((0, b''), self._generate_keywords(os.devnull, - TEST_PY_FILE)) - self.assertTrue(filecmp.cmp(TEST_PY_FILE, PY_FILE_WITHOUT_KEYWORDS)) - - def test_keywords_py_without_markers_produces_error(self): - rc, stderr = self._generate_keywords(os.devnull, os.devnull) - self.assertNotEqual(rc, 0) - self.assertRegex(stderr, b'does not contain format markers') - - def test_missing_grammar_file_produces_error(self): - rc, stderr = self._generate_keywords(NONEXISTENT_FILE, KEYWORD_FILE) - self.assertNotEqual(rc, 0) - self.assertRegex(stderr, b'(?ms)' + NONEXISTENT_FILE.encode()) + def test_async_and_await_are_keywords(self): + self.assertIn("async", keyword.kwlist) + self.assertIn("await", keyword.kwlist) - def test_missing_keywords_py_file_produces_error(self): - rc, stderr = self._generate_keywords(os.devnull, NONEXISTENT_FILE) - self.assertNotEqual(rc, 0) - self.assertRegex(stderr, b'(?ms)' + NONEXISTENT_FILE.encode()) + def test_keywords_are_sorted(self): + self.assertListEqual(sorted(keyword.kwlist), keyword.kwlist) if __name__ == "__main__": diff --git a/Makefile.pre.in b/Makefile.pre.in index 8042e8e81a0564..174b12c5de8bcf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -724,7 +724,7 @@ regen-importlib: Programs/_freeze_importlib # Regenerate all generated files regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar \ - regen-token regen-symbol regen-ast regen-importlib clinic + regen-token regen-keyword regen-symbol regen-ast regen-importlib clinic ############################################################################ # Special rules for object files @@ -843,6 +843,15 @@ regen-token: $(srcdir)/Grammar/Tokens \ $(srcdir)/Lib/token.py +.PHONY: regen-keyword +regen-keyword: + # Regenerate Lib/keyword.py from Grammar/Grammar and Grammar/Tokens + # using Parser/pgen + $(PYTHON_FOR_REGEN) -m Parser.pgen.keywordgen $(srcdir)/Grammar/Grammar \ + $(srcdir)/Grammar/Tokens \ + $(srcdir)/Lib/keyword.py.new + $(UPDATE_FILE) $(srcdir)/Lib/keyword.py $(srcdir)/Lib/keyword.py.new + .PHONY: regen-symbol regen-symbol: $(srcdir)/Include/graminit.h # Regenerate Lib/symbol.py from Include/graminit.h diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-03-20-00-37-24.bpo-12456.fnKoKo.rst b/Misc/NEWS.d/next/Core and Builtins/2019-03-20-00-37-24.bpo-12456.fnKoKo.rst new file mode 100644 index 00000000000000..10d6c496273476 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-03-20-00-37-24.bpo-12456.fnKoKo.rst @@ -0,0 +1,2 @@ +Regenerate :mod:`keyword` from the Grammar and Tokens file using pgen. Patch +by Pablo Galindo. diff --git a/Parser/pgen/keywordgen.py b/Parser/pgen/keywordgen.py new file mode 100644 index 00000000000000..fbae53bdc8d8f3 --- /dev/null +++ b/Parser/pgen/keywordgen.py @@ -0,0 +1,62 @@ +"""Generate Lib/keyword.py from the Grammar and Tokens files using pgen""" + +import argparse + +from .pgen import ParserGenerator + +TEMPLATE = '''\ +#! /usr/bin/env python3 + +"""Keywords (from "Grammar/Grammar") + +This file is automatically generated; please don't muck it up! + +To update the symbols in this file, 'cd' to the top directory of +the python source tree and run: + + python -m Parser.pgen.keywordgen ./Grammar/Grammar \\ + ./Grammar/Tokens \\ + ./Lib/keyword.py + +Alternatively, you can run 'make regen-keyword'. +""" + +__all__ = ["iskeyword", "kwlist"] + +kwlist = [ +{keywords} + ] + +iskeyword = frozenset(kwlist).__contains__ +''' + +EXTRA_KEYWORDS = ["async", "await"] + + +def main(): + parser = argparse.ArgumentParser(description="Parser generator main program.") + parser.add_argument( + "grammar", type=str, help="The file with the grammar definition in EBNF format" + ) + parser.add_argument( + "tokens", type=str, help="The file with the token definitions" + ) + parser.add_argument( + "keyword_file", + type=argparse.FileType('w'), + help="The path to write the keyword definitions", + ) + args = parser.parse_args() + p = ParserGenerator(args.grammar, args.tokens) + grammar = p.make_grammar() + + with args.keyword_file as thefile: + all_keywords = sorted(list(grammar.keywords) + EXTRA_KEYWORDS) + + keywords = "\n".join(" '{}',".format(keyword) + for keyword in all_keywords) + thefile.write(TEMPLATE.format(keywords=keywords)) + + +if __name__ == "__main__": + main() From bffdb57e42f70fc0f2f341e35668b72f53699ea3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 21 Mar 2019 00:33:31 +0000 Subject: [PATCH 2/5] Implement Victor's suggestions --- Lib/keyword.py | 72 +++++++++++++++++++-------------------- Parser/pgen/keywordgen.py | 15 ++++---- 2 files changed, 43 insertions(+), 44 deletions(-) diff --git a/Lib/keyword.py b/Lib/keyword.py index 605d7504abad54..c6dbbbb1c149d3 100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -1,3 +1,4 @@ +\ #! /usr/bin/env python3 """Keywords (from "Grammar/Grammar") @@ -17,41 +18,40 @@ __all__ = ["iskeyword", "kwlist"] kwlist = [ - 'False', - 'None', - 'True', - 'and', - 'as', - 'assert', - 'async', - 'await', - 'break', - 'class', - 'continue', - 'def', - 'del', - 'elif', - 'else', - 'except', - 'finally', - 'for', - 'from', - 'global', - 'if', - 'import', - 'in', - 'is', - 'lambda', - 'nonlocal', - 'not', - 'or', - 'pass', - 'raise', - 'return', - 'try', - 'while', - 'with', - 'yield', - ] + 'False', + 'None', + 'True', + 'and', + 'as', + 'assert', + 'async', + 'await', + 'break', + 'class', + 'continue', + 'def', + 'del', + 'elif', + 'else', + 'except', + 'finally', + 'for', + 'from', + 'global', + 'if', + 'import', + 'in', + 'is', + 'lambda', + 'nonlocal', + 'not', + 'or', + 'pass', + 'raise', + 'return', + 'try', + 'while', + 'with', + 'yield'] iskeyword = frozenset(kwlist).__contains__ diff --git a/Parser/pgen/keywordgen.py b/Parser/pgen/keywordgen.py index fbae53bdc8d8f3..f56f0b9fbe28a1 100644 --- a/Parser/pgen/keywordgen.py +++ b/Parser/pgen/keywordgen.py @@ -4,7 +4,7 @@ from .pgen import ParserGenerator -TEMPLATE = '''\ +TEMPLATE = r'''\ #! /usr/bin/env python3 """Keywords (from "Grammar/Grammar") @@ -14,8 +14,8 @@ To update the symbols in this file, 'cd' to the top directory of the python source tree and run: - python -m Parser.pgen.keywordgen ./Grammar/Grammar \\ - ./Grammar/Tokens \\ + python -m Parser.pgen.keywordgen ./Grammar/Grammar \ + ./Grammar/Tokens \ ./Lib/keyword.py Alternatively, you can run 'make regen-keyword'. @@ -24,8 +24,7 @@ __all__ = ["iskeyword", "kwlist"] kwlist = [ -{keywords} - ] + {keywords}] iskeyword = frozenset(kwlist).__contains__ ''' @@ -53,9 +52,9 @@ def main(): with args.keyword_file as thefile: all_keywords = sorted(list(grammar.keywords) + EXTRA_KEYWORDS) - keywords = "\n".join(" '{}',".format(keyword) - for keyword in all_keywords) - thefile.write(TEMPLATE.format(keywords=keywords)) + keywords = ",\n ".join("'{}'".format(keyword) + for keyword in all_keywords) + thefile.write(TEMPLATE.lstrip().format(keywords=keywords)) if __name__ == "__main__": From 959cee26235b2c70a74bd8844b735f02b2c30c9f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 21 Mar 2019 00:53:19 +0000 Subject: [PATCH 3/5] fixup! Implement Victor's suggestions --- Lib/keyword.py | 2 +- Parser/pgen/keywordgen.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/keyword.py b/Lib/keyword.py index c6dbbbb1c149d3..9f233579b9e087 100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -8,7 +8,7 @@ To update the symbols in this file, 'cd' to the top directory of the python source tree and run: - python -m Parser.pgen.keywordgen ./Grammar/Grammar \ + python3 -m Parser.pgen.keywordgen ./Grammar/Grammar \ ./Grammar/Tokens \ ./Lib/keyword.py diff --git a/Parser/pgen/keywordgen.py b/Parser/pgen/keywordgen.py index f56f0b9fbe28a1..aba3d681ac3e2d 100644 --- a/Parser/pgen/keywordgen.py +++ b/Parser/pgen/keywordgen.py @@ -14,7 +14,7 @@ To update the symbols in this file, 'cd' to the top directory of the python source tree and run: - python -m Parser.pgen.keywordgen ./Grammar/Grammar \ + python3 -m Parser.pgen.keywordgen ./Grammar/Grammar \ ./Grammar/Tokens \ ./Lib/keyword.py @@ -27,7 +27,7 @@ {keywords}] iskeyword = frozenset(kwlist).__contains__ -''' +'''.lstrip() EXTRA_KEYWORDS = ["async", "await"] @@ -54,7 +54,7 @@ def main(): keywords = ",\n ".join("'{}'".format(keyword) for keyword in all_keywords) - thefile.write(TEMPLATE.lstrip().format(keywords=keywords)) + thefile.write(TEMPLATE.format(keywords=keywords)) if __name__ == "__main__": From 78cbbf0cb43324417ef259a59dc9115108e834a5 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 21 Mar 2019 10:01:50 +0000 Subject: [PATCH 4/5] fixup! fixup! Implement Victor's suggestions --- Lib/keyword.py | 7 +++---- Parser/pgen/keywordgen.py | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Lib/keyword.py b/Lib/keyword.py index 9f233579b9e087..e11fc365a66b5b 100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -1,4 +1,3 @@ -\ #! /usr/bin/env python3 """Keywords (from "Grammar/Grammar") @@ -8,9 +7,9 @@ To update the symbols in this file, 'cd' to the top directory of the python source tree and run: - python3 -m Parser.pgen.keywordgen ./Grammar/Grammar \ - ./Grammar/Tokens \ - ./Lib/keyword.py + python3 -m Parser.pgen.keywordgen Grammar/Grammar \ + Grammar/Tokens \ + Lib/keyword.py Alternatively, you can run 'make regen-keyword'. """ diff --git a/Parser/pgen/keywordgen.py b/Parser/pgen/keywordgen.py index aba3d681ac3e2d..a32f451d5baac4 100644 --- a/Parser/pgen/keywordgen.py +++ b/Parser/pgen/keywordgen.py @@ -4,7 +4,7 @@ from .pgen import ParserGenerator -TEMPLATE = r'''\ +TEMPLATE = r''' #! /usr/bin/env python3 """Keywords (from "Grammar/Grammar") @@ -14,9 +14,9 @@ To update the symbols in this file, 'cd' to the top directory of the python source tree and run: - python3 -m Parser.pgen.keywordgen ./Grammar/Grammar \ - ./Grammar/Tokens \ - ./Lib/keyword.py + python3 -m Parser.pgen.keywordgen Grammar/Grammar \ + Grammar/Tokens \ + Lib/keyword.py Alternatively, you can run 'make regen-keyword'. """ From c17c209525bebe289f958e1b7cfdcbb03cc2ffc9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 21 Mar 2019 16:14:36 +0000 Subject: [PATCH 5/5] fixup! fixup! fixup! Implement Victor's suggestions --- Lib/keyword.py | 5 ++--- Parser/pgen/keywordgen.py | 11 +++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Lib/keyword.py b/Lib/keyword.py index e11fc365a66b5b..ddcbb25d3d3f58 100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Keywords (from "Grammar/Grammar") This file is automatically generated; please don't muck it up! @@ -51,6 +49,7 @@ 'try', 'while', 'with', - 'yield'] + 'yield' +] iskeyword = frozenset(kwlist).__contains__ diff --git a/Parser/pgen/keywordgen.py b/Parser/pgen/keywordgen.py index a32f451d5baac4..eeb3ef739fadef 100644 --- a/Parser/pgen/keywordgen.py +++ b/Parser/pgen/keywordgen.py @@ -5,8 +5,6 @@ from .pgen import ParserGenerator TEMPLATE = r''' -#! /usr/bin/env python3 - """Keywords (from "Grammar/Grammar") This file is automatically generated; please don't muck it up! @@ -24,7 +22,8 @@ __all__ = ["iskeyword", "kwlist"] kwlist = [ - {keywords}] + {keywords} +] iskeyword = frozenset(kwlist).__contains__ '''.lstrip() @@ -33,7 +32,8 @@ def main(): - parser = argparse.ArgumentParser(description="Parser generator main program.") + parser = argparse.ArgumentParser(description="Generate the Lib/keywords.py " + "file from the grammar.") parser.add_argument( "grammar", type=str, help="The file with the grammar definition in EBNF format" ) @@ -52,8 +52,7 @@ def main(): with args.keyword_file as thefile: all_keywords = sorted(list(grammar.keywords) + EXTRA_KEYWORDS) - keywords = ",\n ".join("'{}'".format(keyword) - for keyword in all_keywords) + keywords = ",\n ".join(map(repr, all_keywords)) thefile.write(TEMPLATE.format(keywords=keywords))