New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
bpo-36143: Regenerate Lib/keyword.py from the Grammar and Tokens file using pgen #12456
Changes from all commits
c2843c9
bffdb57
959cee2
78cbbf0
c17c209
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,98 +1,55 @@ | ||
#! /usr/bin/env python3 | ||
|
||
"""Keywords (from "graminit.c") | ||
"""Keywords (from "Grammar/Grammar") | ||
|
||
This file is automatically generated; please don't muck it up! | ||
|
||
To update the symbols in this file, 'cd' to the top directory of | ||
the python source tree after building the interpreter and run: | ||
the python source tree and run: | ||
|
||
python3 -m Parser.pgen.keywordgen Grammar/Grammar \ | ||
Grammar/Tokens \ | ||
Lib/keyword.py | ||
|
||
./python Lib/keyword.py | ||
Alternatively, you can run 'make regen-keyword'. | ||
""" | ||
|
||
__all__ = ["iskeyword", "kwlist"] | ||
|
||
kwlist = [ | ||
#--start keywords-- | ||
'False', | ||
'None', | ||
'True', | ||
'and', | ||
'as', | ||
'assert', | ||
'break', | ||
'class', | ||
'continue', | ||
'def', | ||
'del', | ||
'elif', | ||
'else', | ||
'except', | ||
'finally', | ||
'for', | ||
'from', | ||
'global', | ||
'if', | ||
'import', | ||
'in', | ||
'is', | ||
'lambda', | ||
'nonlocal', | ||
'not', | ||
'or', | ||
'pass', | ||
'raise', | ||
'return', | ||
'try', | ||
'while', | ||
'with', | ||
'yield', | ||
#--end keywords-- | ||
] | ||
|
||
kwlist.append('async') | ||
kwlist.append('await') | ||
kwlist.sort() | ||
'False', | ||
'None', | ||
'True', | ||
'and', | ||
'as', | ||
'assert', | ||
'async', | ||
'await', | ||
'break', | ||
'class', | ||
'continue', | ||
'def', | ||
'del', | ||
'elif', | ||
'else', | ||
'except', | ||
'finally', | ||
'for', | ||
'from', | ||
'global', | ||
'if', | ||
'import', | ||
'in', | ||
'is', | ||
'lambda', | ||
'nonlocal', | ||
'not', | ||
'or', | ||
'pass', | ||
'raise', | ||
'return', | ||
'try', | ||
'while', | ||
'with', | ||
'yield' | ||
] | ||
|
||
iskeyword = frozenset(kwlist).__contains__ | ||
|
||
def main(): | ||
import sys, re | ||
|
||
args = sys.argv[1:] | ||
iptfile = args and args[0] or "Python/graminit.c" | ||
if len(args) > 1: optfile = args[1] | ||
else: optfile = "Lib/keyword.py" | ||
|
||
# load the output skeleton from the target, taking care to preserve its | ||
# newline convention. | ||
with open(optfile, newline='') as fp: | ||
format = fp.readlines() | ||
nl = format[0][len(format[0].strip()):] if format else '\n' | ||
|
||
# scan the source file for keywords | ||
with open(iptfile) as fp: | ||
strprog = re.compile('"([^"]+)"') | ||
lines = [] | ||
for line in fp: | ||
if '{1, "' in line: | ||
match = strprog.search(line) | ||
if match: | ||
lines.append(" '" + match.group(1) + "'," + nl) | ||
lines.sort() | ||
|
||
# insert the lines of keywords into the skeleton | ||
try: | ||
start = format.index("#--start keywords--" + nl) + 1 | ||
end = format.index("#--end keywords--" + nl) | ||
format[start:end] = lines | ||
except ValueError: | ||
sys.stderr.write("target does not contain format markers\n") | ||
sys.exit(1) | ||
|
||
# write the output file | ||
with open(optfile, 'w', newline='') as fp: | ||
fp.writelines(format) | ||
|
||
if __name__ == "__main__": | ||
main() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
Regenerate :mod:`keyword` from the Grammar and Tokens file using pgen. Patch | ||
by Pablo Galindo. |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
"""Generate Lib/keyword.py from the Grammar and Tokens files using pgen""" | ||
|
||
import argparse | ||
|
||
from .pgen import ParserGenerator | ||
|
||
TEMPLATE = r''' | ||
"""Keywords (from "Grammar/Grammar") | ||
|
||
This file is automatically generated; please don't muck it up! | ||
|
||
To update the symbols in this file, 'cd' to the top directory of | ||
the python source tree and run: | ||
|
||
python3 -m Parser.pgen.keywordgen Grammar/Grammar \ | ||
Grammar/Tokens \ | ||
Lib/keyword.py | ||
|
||
Alternatively, you can run 'make regen-keyword'. | ||
vstinner marked this conversation as resolved.
Show resolved
Hide resolved
|
||
""" | ||
|
||
__all__ = ["iskeyword", "kwlist"] | ||
|
||
kwlist = [ | ||
{keywords} | ||
] | ||
|
||
iskeyword = frozenset(kwlist).__contains__ | ||
'''.lstrip() | ||
|
||
EXTRA_KEYWORDS = ["async", "await"] | ||
|
||
|
||
def main(): | ||
parser = argparse.ArgumentParser(description="Generate the Lib/keywords.py " | ||
"file from the grammar.") | ||
parser.add_argument( | ||
"grammar", type=str, help="The file with the grammar definition in EBNF format" | ||
) | ||
parser.add_argument( | ||
"tokens", type=str, help="The file with the token definitions" | ||
serhiy-storchaka marked this conversation as resolved.
Show resolved
Hide resolved
|
||
) | ||
parser.add_argument( | ||
"keyword_file", | ||
type=argparse.FileType('w'), | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would be nice to make the script working on read-only sources (doing nothing if the file is not changed). See for example There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sorry, I am not sure I understand exactly what do you mean :( Can you elaborate a bit more? Or if you want to do it directly, please, push directly to my branch to update the PR :) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. See "make regen-opcode": Antoine Pitrou wrote $(UPDATE_FILE) which leaves the file unchanged if the content didn't change. One advantage is to not touch the modification time of the file at all. It matters for Makefile to avoid useless recompilation. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I propose to use function There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Although I like |
||
help="The path to write the keyword definitions", | ||
) | ||
args = parser.parse_args() | ||
p = ParserGenerator(args.grammar, args.tokens) | ||
grammar = p.make_grammar() | ||
|
||
with args.keyword_file as thefile: | ||
all_keywords = sorted(list(grammar.keywords) + EXTRA_KEYWORDS) | ||
|
||
keywords = ",\n ".join(map(repr, all_keywords)) | ||
thefile.write(TEMPLATE.format(keywords=keywords)) | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@serhiy-storchaka: Sorry, I'm the one who asked @pablogsal to write
'yield']
. It doesn't matter, I'm fine with "]" on a separated line as well.