forked from python/cpython
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
bpo-36143: Regenerate Lib/keyword.py from the Grammar and Tokens file…
… using pgen (pythonGH-12456) Now that the parser generator is written in Python (Parser/pgen) we can make use of it to regenerate the Lib/keyword file that contains the language keywords instead of parsing the autogenerated grammar files. This also allows checking in the CI that the autogenerated files are up to date.
- Loading branch information
Showing
5 changed files
with
124 additions
and
197 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,98 +1,55 @@ | ||
| #! /usr/bin/env python3 | ||
|
|
||
| """Keywords (from "graminit.c") | ||
| """Keywords (from "Grammar/Grammar") | ||
| This file is automatically generated; please don't muck it up! | ||
| To update the symbols in this file, 'cd' to the top directory of | ||
| the python source tree after building the interpreter and run: | ||
| the python source tree and run: | ||
| python3 -m Parser.pgen.keywordgen Grammar/Grammar \ | ||
| Grammar/Tokens \ | ||
| Lib/keyword.py | ||
| ./python Lib/keyword.py | ||
| Alternatively, you can run 'make regen-keyword'. | ||
| """ | ||
|
|
||
| __all__ = ["iskeyword", "kwlist"] | ||
|
|
||
| kwlist = [ | ||
| #--start keywords-- | ||
| 'False', | ||
| 'None', | ||
| 'True', | ||
| 'and', | ||
| 'as', | ||
| 'assert', | ||
| 'break', | ||
| 'class', | ||
| 'continue', | ||
| 'def', | ||
| 'del', | ||
| 'elif', | ||
| 'else', | ||
| 'except', | ||
| 'finally', | ||
| 'for', | ||
| 'from', | ||
| 'global', | ||
| 'if', | ||
| 'import', | ||
| 'in', | ||
| 'is', | ||
| 'lambda', | ||
| 'nonlocal', | ||
| 'not', | ||
| 'or', | ||
| 'pass', | ||
| 'raise', | ||
| 'return', | ||
| 'try', | ||
| 'while', | ||
| 'with', | ||
| 'yield', | ||
| #--end keywords-- | ||
| ] | ||
|
|
||
| kwlist.append('async') | ||
| kwlist.append('await') | ||
| kwlist.sort() | ||
| 'False', | ||
| 'None', | ||
| 'True', | ||
| 'and', | ||
| 'as', | ||
| 'assert', | ||
| 'async', | ||
| 'await', | ||
| 'break', | ||
| 'class', | ||
| 'continue', | ||
| 'def', | ||
| 'del', | ||
| 'elif', | ||
| 'else', | ||
| 'except', | ||
| 'finally', | ||
| 'for', | ||
| 'from', | ||
| 'global', | ||
| 'if', | ||
| 'import', | ||
| 'in', | ||
| 'is', | ||
| 'lambda', | ||
| 'nonlocal', | ||
| 'not', | ||
| 'or', | ||
| 'pass', | ||
| 'raise', | ||
| 'return', | ||
| 'try', | ||
| 'while', | ||
| 'with', | ||
| 'yield' | ||
| ] | ||
|
|
||
| iskeyword = frozenset(kwlist).__contains__ | ||
|
|
||
| def main(): | ||
| import sys, re | ||
|
|
||
| args = sys.argv[1:] | ||
| iptfile = args and args[0] or "Python/graminit.c" | ||
| if len(args) > 1: optfile = args[1] | ||
| else: optfile = "Lib/keyword.py" | ||
|
|
||
| # load the output skeleton from the target, taking care to preserve its | ||
| # newline convention. | ||
| with open(optfile, newline='') as fp: | ||
| format = fp.readlines() | ||
| nl = format[0][len(format[0].strip()):] if format else '\n' | ||
|
|
||
| # scan the source file for keywords | ||
| with open(iptfile) as fp: | ||
| strprog = re.compile('"([^"]+)"') | ||
| lines = [] | ||
| for line in fp: | ||
| if '{1, "' in line: | ||
| match = strprog.search(line) | ||
| if match: | ||
| lines.append(" '" + match.group(1) + "'," + nl) | ||
| lines.sort() | ||
|
|
||
| # insert the lines of keywords into the skeleton | ||
| try: | ||
| start = format.index("#--start keywords--" + nl) + 1 | ||
| end = format.index("#--end keywords--" + nl) | ||
| format[start:end] = lines | ||
| except ValueError: | ||
| sys.stderr.write("target does not contain format markers\n") | ||
| sys.exit(1) | ||
|
|
||
| # write the output file | ||
| with open(optfile, 'w', newline='') as fp: | ||
| fp.writelines(format) | ||
|
|
||
| if __name__ == "__main__": | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 2 additions & 0 deletions
2
Misc/NEWS.d/next/Core and Builtins/2019-03-20-00-37-24.bpo-12456.fnKoKo.rst
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| Regenerate :mod:`keyword` from the Grammar and Tokens file using pgen. Patch | ||
| by Pablo Galindo. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| """Generate Lib/keyword.py from the Grammar and Tokens files using pgen""" | ||
|
|
||
| import argparse | ||
|
|
||
| from .pgen import ParserGenerator | ||
|
|
||
| TEMPLATE = r''' | ||
| """Keywords (from "Grammar/Grammar") | ||
| This file is automatically generated; please don't muck it up! | ||
| To update the symbols in this file, 'cd' to the top directory of | ||
| the python source tree and run: | ||
| python3 -m Parser.pgen.keywordgen Grammar/Grammar \ | ||
| Grammar/Tokens \ | ||
| Lib/keyword.py | ||
| Alternatively, you can run 'make regen-keyword'. | ||
| """ | ||
| __all__ = ["iskeyword", "kwlist"] | ||
| kwlist = [ | ||
| {keywords} | ||
| ] | ||
| iskeyword = frozenset(kwlist).__contains__ | ||
| '''.lstrip() | ||
|
|
||
| EXTRA_KEYWORDS = ["async", "await"] | ||
|
|
||
|
|
||
| def main(): | ||
| parser = argparse.ArgumentParser(description="Generate the Lib/keywords.py " | ||
| "file from the grammar.") | ||
| parser.add_argument( | ||
| "grammar", type=str, help="The file with the grammar definition in EBNF format" | ||
| ) | ||
| parser.add_argument( | ||
| "tokens", type=str, help="The file with the token definitions" | ||
| ) | ||
| parser.add_argument( | ||
| "keyword_file", | ||
| type=argparse.FileType('w'), | ||
| help="The path to write the keyword definitions", | ||
| ) | ||
| args = parser.parse_args() | ||
| p = ParserGenerator(args.grammar, args.tokens) | ||
| grammar = p.make_grammar() | ||
|
|
||
| with args.keyword_file as thefile: | ||
| all_keywords = sorted(list(grammar.keywords) + EXTRA_KEYWORDS) | ||
|
|
||
| keywords = ",\n ".join(map(repr, all_keywords)) | ||
| thefile.write(TEMPLATE.format(keywords=keywords)) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() |