From 9a221402e6086678c921b77eef4b676cbdefb8ae Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 6 Aug 2016 10:40:35 +0200 Subject: [PATCH] Make Python utils more idiomatic, use better names, fix -c mode of update_wiki. --- .gitignore | 1 + util/export.py | 156 ++++++++------------------------------------ util/lintlib.py | 95 +++++++++++++++++++++++++++ util/update_wiki.py | 154 ++++++++++--------------------------------- 4 files changed, 160 insertions(+), 246 deletions(-) create mode 100644 util/lintlib.py diff --git a/.gitignore b/.gitignore index 0206d9dc96365..a6b636709c523 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ out *.so *.rlib *.dll +*.pyc # Executables *.exe diff --git a/util/export.py b/util/export.py index 8d95c70f1e777..ae8e4a72c08d8 100755 --- a/util/export.py +++ b/util/export.py @@ -1,79 +1,29 @@ #!/usr/bin/env python # Build the gh-pages -import json -import os import re import sys +import json +from lintlib import parse_all, log -level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''') -conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE) -confvar_re = re.compile(r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''') lint_subheadline = re.compile(r'''^\*\*([\w\s]+?)[:?.!]?\*\*(.*)''') -conf_template = """ +CONF_TEMPLATE = """\ This lint has the following configuration variables: -* `%s: %s`: %s (defaults to `%s`). -""" - - -# TODO: actual logging -def warn(*args): - print(args) - - -def debug(*args): - print(args) - - -def info(*args): - print(args) - - -def parse_path(p="clippy_lints/src"): - lints = [] - for f in os.listdir(p): - if f.endswith(".rs"): - parse_file(lints, os.path.join(p, f)) - - conf = parse_conf(p) - info(conf) +* `%s: %s`: %s (defaults to `%s`).""" - for lint_id in conf: - lint = next(l for l in lints if l['id'] == lint_id) - if lint: - lint['docs']['Configuration'] = (conf_template % conf[lint_id]).strip() - return lints - - -def parse_conf(p): - c = {} - with open(p + '/utils/conf.rs') as f: - f = f.read() - - m = re.search(conf_re, f) - m = m.groups()[0] - - m = re.findall(confvar_re, m) - - for (lint, doc, name, default, ty) in m: - c[lint.lower()] = (name, ty, doc, default) - - return c - - -def parseLintDef(level, comment, name): - lint = {} - lint['id'] = name - lint['level'] = level - lint['docs'] = {} +def parse_lint_def(lint): + lint_dict = {} + lint_dict['id'] = lint.name + lint_dict['level'] = lint.level + lint_dict['docs'] = {} last_section = None - for line in comment: + for line in lint.doc: if len(line.strip()) == 0: continue @@ -86,77 +36,29 @@ def parseLintDef(level, comment, name): text = line if not last_section: - warn("Skipping comment line as it was not preceded by a heading") - debug("in lint `%s`, line `%s`" % name, line) - - lint['docs'][last_section] = (lint['docs'].get(last_section, "") + "\n" + text).strip() - - return lint - - -def parse_file(d, f): - last_comment = [] - comment = True - - with open(f) as rs: - for line in rs: - if comment: - if line.startswith("///"): - if line.startswith("/// "): - last_comment.append(line[4:]) - else: - last_comment.append(line[3:]) - elif line.startswith("declare_lint!"): - comment = False - deprecated = False - restriction = False - elif line.startswith("declare_restriction_lint!"): - comment = False - deprecated = False - restriction = True - elif line.startswith("declare_deprecated_lint!"): - comment = False - deprecated = True - else: - last_comment = [] - if not comment: - l = line.strip() - m = re.search(r"pub\s+([A-Z_][A-Z_0-9]*)", l) - - if m: - name = m.group(1).lower() - - # Intentionally either a never looping or infinite loop - while not deprecated and not restriction: - m = re.search(level_re, line) - if m: - level = m.group(0) - break - - line = next(rs) - - if deprecated: - level = "Deprecated" - elif restriction: - level = "Allow" - - info("found %s with level %s in %s" % (name, level, f)) - d.append(parseLintDef(level, last_comment, name=name)) - last_comment = [] - comment = True - if "}" in l: - warn("Warning: Missing Lint-Name in", f) - comment = True + log.warn("Skipping comment line as it was not preceded by a heading") + log.debug("in lint `%s`, line `%s`", lint.name, line) + + lint_dict['docs'][last_section] = \ + (lint_dict['docs'].get(last_section, "") + "\n" + text).strip() + + return lint_dict def main(): - lints = parse_path() - info("got %s lints" % len(lints)) + lintlist, configs = parse_all() + lints = {} + for lint in lintlist: + lints[lint.name] = parse_lint_def(lint) + if lint.name in configs: + lints[lint.name]['docs']['Configuration'] = \ + CONF_TEMPLATE % configs[lint.name] + + outfile = sys.argv[1] if len(sys.argv) > 1 else "util/gh-pages/lints.json" + with open(outfile, "w") as fp: + json.dump(list(lints.values()), fp, indent=2) + log.info("wrote JSON for great justice") - outdir = sys.argv[1] if len(sys.argv) > 1 else "util/gh-pages/lints.json" - with open(outdir, "w") as file: - json.dump(lints, file, indent=2) - info("wrote JSON for great justice") if __name__ == "__main__": main() diff --git a/util/lintlib.py b/util/lintlib.py new file mode 100644 index 0000000000000..45a7110b6a977 --- /dev/null +++ b/util/lintlib.py @@ -0,0 +1,95 @@ +# Common utils for the several housekeeping scripts. + +import os +import re +import collections + +import logging as log +log.basicConfig(level=log.INFO, format='%(levelname)s: %(message)s') + +Lint = collections.namedtuple('Lint', 'name level doc sourcefile') +Config = collections.namedtuple('Config', 'name ty doc default') + +lintname_re = re.compile(r'''pub\s+([A-Z_][A-Z_0-9]*)''') +level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''') +conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE) +confvar_re = re.compile( + r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''') + + +def parse_lints(lints, filepath): + last_comment = [] + comment = True + + with open(filepath) as fp: + for line in fp: + if comment: + if line.startswith("/// "): + last_comment.append(line[4:]) + elif line.startswith("///"): + last_comment.append(line[3:]) + elif line.startswith("declare_lint!"): + comment = False + deprecated = False + restriction = False + elif line.startswith("declare_restriction_lint!"): + comment = False + deprecated = False + restriction = True + elif line.startswith("declare_deprecated_lint!"): + comment = False + deprecated = True + else: + last_comment = [] + if not comment: + m = lintname_re.search(line) + if m: + name = m.group(1).lower() + + if deprecated: + level = "Deprecated" + elif restriction: + level = "Allow" + else: + while True: + m = level_re.search(line) + if m: + level = m.group(0) + break + line = next(fp) + + log.info("found %s with level %s in %s", + name, level, filepath) + lints.append(Lint(name, level, last_comment, filepath)) + last_comment = [] + comment = True + if "}" in line: + log.warn("Warning: missing Lint-Name in %s", filepath) + comment = True + + +def parse_configs(path): + configs = {} + with open(os.path.join(path, 'utils/conf.rs')) as fp: + contents = fp.read() + + match = re.search(conf_re, contents) + confvars = re.findall(confvar_re, match.group(1)) + + for (lint, doc, name, default, ty) in confvars: + configs[lint.lower()] = Config(name, ty, doc, default) + + return configs + + +def parse_all(path="clippy_lints/src"): + lints = [] + for filename in os.listdir(path): + if filename.endswith(".rs"): + parse_lints(lints, os.path.join(path, filename)) + log.info("got %s lints", len(lints)) + + configs = parse_configs(path) + log.info("got %d configs", len(configs)) + + return lints, configs diff --git a/util/update_wiki.py b/util/update_wiki.py index e8b2280cb4716..a9bd32f209897 100755 --- a/util/update_wiki.py +++ b/util/update_wiki.py @@ -3,93 +3,11 @@ # requires the checked out wiki in ../rust-clippy.wiki/ # with -c option, print a warning and set exit status 1 if the file would be # changed. -import os + import re import sys - -level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''') -conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE) -confvar_re = re.compile(r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''') - - -def parse_path(p="clippy_lints/src"): - d = {} - for f in os.listdir(p): - if f.endswith(".rs"): - parse_file(d, os.path.join(p, f)) - return (d, parse_conf(p)) - - -def parse_conf(p): - c = {} - with open(p + '/utils/conf.rs') as f: - f = f.read() - - m = re.search(conf_re, f) - m = m.groups()[0] - - m = re.findall(confvar_re, m) - - for (lint, doc, name, default, ty) in m: - c[lint.lower()] = (name, ty, doc, default) - - return c - - -def parse_file(d, f): - last_comment = [] - comment = True - - with open(f) as rs: - for line in rs: - if comment: - if line.startswith("///"): - if line.startswith("/// "): - last_comment.append(line[4:]) - else: - last_comment.append(line[3:]) - elif line.startswith("declare_lint!"): - comment = False - deprecated = False - restriction = False - elif line.startswith("declare_restriction_lint!"): - comment = False - deprecated = False - restriction = True - elif line.startswith("declare_deprecated_lint!"): - comment = False - deprecated = True - else: - last_comment = [] - if not comment: - l = line.strip() - m = re.search(r"pub\s+([A-Z_][A-Z_0-9]*)", l) - - if m: - name = m.group(1).lower() - - # Intentionally either a never looping or infinite loop - while not deprecated and not restriction: - m = re.search(level_re, line) - if m: - level = m.group(0) - break - - line = next(rs) - - if deprecated: - level = "Deprecated" - elif restriction: - level = "Allow" - - print("found %s with level %s in %s" % (name, level, f)) - d[name] = (level, last_comment) - last_comment = [] - comment = True - if "}" in l: - print("Warning: Missing Lint-Name in", f) - comment = True +from lintlib import log, parse_all PREFIX = """Welcome to the rust-clippy wiki! @@ -108,14 +26,13 @@ def parse_file(d, f): """ - -template = """\n# `%s` +TEMPLATE = """\n# `%s` **Default level:** %s %s""" -conf_template = """ +CONF_TEMPLATE = """ **Configuration:** This lint has the following configuration variables: * `%s: %s`: %s (defaults to `%s`). @@ -129,51 +46,50 @@ def level_message(level): return "\n**Those lints are %s by default**:\n\n" % level -def write_wiki_page(d, c, f): - keys = list(d.keys()) - keys.sort() - with open(f, "w") as w: - w.write(PREFIX) +def write_wiki_page(lints, configs, filepath): + lints.sort() + with open(filepath, "w") as fp: + fp.write(PREFIX) for level in ('Deny', 'Warn', 'Allow', 'Deprecated'): - w.write(level_message(level)) - for k in keys: - if d[k][0] == level: - w.write("[`%s`](#%s)\n" % (k, k)) + fp.write(level_message(level)) + for lint in lints: + if lint.level == level: + fp.write("[`%s`](#%s)\n" % (lint.name, lint.name)) - w.write(WARNING) - for k in keys: - w.write(template % (k, d[k][0], "".join(d[k][1]))) + fp.write(WARNING) + for lint in lints: + fp.write(TEMPLATE % (lint.name, lint.level, "".join(lint.doc))) - if k in c: - w.write(conf_template % c[k]) + if lint.name in configs: + fp.write(CONF_TEMPLATE % configs[lint.name]) -def check_wiki_page(d, c, f): - errors = [] - with open(f) as w: - for line in w: - m = re.match("# `([a-z_]+)`", line) +def check_wiki_page(lints, configs, filepath): + lintdict = dict((lint.name, lint) for lint in lints) + errors = False + with open(filepath) as fp: + for line in fp: + m = re.match("# `([a-z_0-9]+)`", line) if m: - v = d.pop(m.group(1), "()") - if v == "()": - errors.append("Missing wiki entry: " + m.group(1)) - keys = list(d.keys()) - keys.sort() - for k in keys: - errors.append("Spurious wiki entry: " + k) + v = lintdict.pop(m.group(1), None) + if v is None: + log.error("Spurious wiki entry: %s", m.group(1)) + errors = True + for n in sorted(lintdict): + log.error("Missing wiki entry: %s", n) + errors = True if errors: - print("\n".join(errors)) - sys.exit(1) + return 1 def main(): - (d, c) = parse_path() - print('Found %s lints' % len(d)) + lints, configs = parse_all() if "-c" in sys.argv: - check_wiki_page(d, c, "../rust-clippy.wiki/Home.md") + check_wiki_page(lints, configs, "../rust-clippy.wiki/Home.md") else: - write_wiki_page(d, c, "../rust-clippy.wiki/Home.md") + write_wiki_page(lints, configs, "../rust-clippy.wiki/Home.md") + if __name__ == "__main__": main()