From c19443b698a6c7c0f98e064297f8856486b87506 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 23 Oct 2020 23:00:35 +0200 Subject: [PATCH 01/32] Add generator.py to regenerate crates and add information into the global README --- generator.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 generator.py diff --git a/generator.py b/generator.py new file mode 100755 index 000000000..261ffbf26 --- /dev/null +++ b/generator.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +from os import listdir +from os.path import isfile, isdir, join +from subprocess import call +import sys + +need_rebuild = False + +def update_workspace(): + try: + call(['bash', '-c', 'cd gir && cargo build --release']) + except: + return False + return True + + +def def_check_submodule(submodule_path): + if len(listdir(submodule_path)) != 0: + return False + print('=> Initializing gir submodule...') + call(['bash', '-c', 'git submodule update --init']) + print('<= Done!') + + question = 'Do you want to update gir submodule? [y/N] ' + if sys.version_info[0] < 3: + line = raw_input(question) + else: + line = input(question) + line = line.strip() + if line.lower() == 'y': + print('=> Updating gir submodule...') + call(['bash', '-c', 'cd gir && git reset --hard HEAD && git pull -f origin master']) + print('<= Done!') + return True + return False + + +def build_gir_if_needed(updated_submodule): + if updated_submodule is True or not isfile('./gir/target/release/gir'): + print('=> Building gir...') + if update_workspace() is True: + print('<= Done!') + else: + print('<= Failed...') + return False + return True + + +def regen_crates(path, level=1): + for entry in [f for f in listdir(path)]: + entry_file = join(path, entry) + if isdir(entry_file): + if level == 1 and not regen_crates(entry_file, 2): + return False + elif entry == 'Gir.toml': + print('==> Regenerating "{}"...'.format(entry_file)) + + args = ['./gir/target/release/gir', '-c', entry_file, '-o', path, '-d', 'gir-files'] + if level > 1: + args.append('-m') + args.append('sys') + try: + call(args) + except Exception as err: + print('The following error occurred: {}'.format(err)) + line = input('Do you want to continue? [y/N] ').strip().lower() + if line != 'y': + return False + print('<== Done!') + + +def main(): + def_check_submodule("gir-files") + if not build_gir_if_needed(def_check_submodule("gir")): + return 1 + + print('=> Regenerating crates...') + if not regen_crates("."): + return 1 + call(['cargo', 'fmt']) + print('<= Done!') + print("Don't forget to check if everything has been correctly generated!") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 982fbe4ad0f1abd3ee63d703bc73d62b4b3d308d Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 23 Oct 2020 23:36:38 +0200 Subject: [PATCH 02/32] Greatly improve generator file --- generator.py | 58 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/generator.py b/generator.py index 261ffbf26..a14ab9d8a 100755 --- a/generator.py +++ b/generator.py @@ -2,24 +2,39 @@ from os import listdir from os.path import isfile, isdir, join -from subprocess import call +import subprocess import sys -need_rebuild = False + +NOTHING_TO_BE_DONE = 0 +NEED_UPDATE = 1 +FAILURE = 2 + + +def run_command(command, folder=None): + if folder is None: + folder = "." + child = subprocess.Popen(command, cwd=folder) + child.communicate() + if child.returncode != 0: + print("Command `{}` failed with return code `{}`...".format(command, child.returncode)) + return False + return True + def update_workspace(): try: - call(['bash', '-c', 'cd gir && cargo build --release']) + return run_command(['cargo', 'build', '--release'], 'gir') except: return False - return True def def_check_submodule(submodule_path): if len(listdir(submodule_path)) != 0: - return False + return NOTHING_TO_BE_DONE print('=> Initializing gir submodule...') - call(['bash', '-c', 'git submodule update --init']) + if not run_command(['git', 'submodule', 'update', '--init']): + return FAILURE print('<= Done!') question = 'Do you want to update gir submodule? [y/N] ' @@ -30,14 +45,19 @@ def def_check_submodule(submodule_path): line = line.strip() if line.lower() == 'y': print('=> Updating gir submodule...') - call(['bash', '-c', 'cd gir && git reset --hard HEAD && git pull -f origin master']) + if not run_command(['git', 'reset', '--hard', 'HEAD'], 'gir'): + return FAILURE + if not run_command(['git', 'pull', '-f', 'origin', 'master'], 'gir'): + return FAILURE print('<= Done!') - return True - return False + return NEED_UPDATE + return NOTHING_TO_BE_DONE def build_gir_if_needed(updated_submodule): - if updated_submodule is True or not isfile('./gir/target/release/gir'): + if updated_submodule == FAILURE: + return False + elif updated_submodule == NEED_UPDATE or not isfile('./gir/target/release/gir'): print('=> Building gir...') if update_workspace() is True: print('<= Done!') @@ -47,11 +67,11 @@ def build_gir_if_needed(updated_submodule): return True -def regen_crates(path, level=1): - for entry in [f for f in listdir(path)]: +def regen_crates(path, level=0): + for entry in listdir(path): entry_file = join(path, entry) if isdir(entry_file): - if level == 1 and not regen_crates(entry_file, 2): + if level < 2 and not regen_crates(entry_file, level + 1): return False elif entry == 'Gir.toml': print('==> Regenerating "{}"...'.format(entry_file)) @@ -60,25 +80,31 @@ def regen_crates(path, level=1): if level > 1: args.append('-m') args.append('sys') + error = False try: - call(args) + error = run_command(args) is False except Exception as err: print('The following error occurred: {}'.format(err)) + error = True + if error is True: line = input('Do you want to continue? [y/N] ').strip().lower() if line != 'y': return False print('<== Done!') + return True def main(): - def_check_submodule("gir-files") + if def_check_submodule("gir-files") == FAILURE: + return 1 if not build_gir_if_needed(def_check_submodule("gir")): return 1 print('=> Regenerating crates...') if not regen_crates("."): return 1 - call(['cargo', 'fmt']) + if not run_command(['cargo', 'fmt']): + return 1 print('<= Done!') print("Don't forget to check if everything has been correctly generated!") return 0 From 6dadb8f897ca8c2910450c894160ca676de2b047 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 6 Nov 2020 14:30:49 +0100 Subject: [PATCH 03/32] Improve generator.py script to make CI job easier --- generator.py | 101 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 17 deletions(-) diff --git a/generator.py b/generator.py index a14ab9d8a..b5f34a5b3 100755 --- a/generator.py +++ b/generator.py @@ -29,7 +29,19 @@ def update_workspace(): return False -def def_check_submodule(submodule_path): +def ask_yes_no_question(question, conf): + question = '{} [y/N] '.format(question) + if conf["yes"] is True: + print(question) + return True + if sys.version_info[0] < 3: + line = raw_input(question) + else: + line = input(question) + return line.strip().lower() == 'y' + + +def def_check_submodule(submodule_path, conf): if len(listdir(submodule_path)) != 0: return NOTHING_TO_BE_DONE print('=> Initializing gir submodule...') @@ -37,13 +49,7 @@ def def_check_submodule(submodule_path): return FAILURE print('<= Done!') - question = 'Do you want to update gir submodule? [y/N] ' - if sys.version_info[0] < 3: - line = raw_input(question) - else: - line = input(question) - line = line.strip() - if line.lower() == 'y': + if ask_yes_no_question('Do you want to update gir submodule?', conf): print('=> Updating gir submodule...') if not run_command(['git', 'reset', '--hard', 'HEAD'], 'gir'): return FAILURE @@ -67,16 +73,16 @@ def build_gir_if_needed(updated_submodule): return True -def regen_crates(path, level=0): +def regen_crates(path, conf, level=0): for entry in listdir(path): entry_file = join(path, entry) if isdir(entry_file): - if level < 2 and not regen_crates(entry_file, level + 1): + if level < 2 and not regen_crates(entry_file, conf, level + 1): return False elif entry == 'Gir.toml': print('==> Regenerating "{}"...'.format(entry_file)) - args = ['./gir/target/release/gir', '-c', entry_file, '-o', path, '-d', 'gir-files'] + args = [conf["gir_path"], '-c', entry_file, '-o', path, '-d', conf["gir_files"]] if level > 1: args.append('-m') args.append('sys') @@ -87,21 +93,82 @@ def regen_crates(path, level=0): print('The following error occurred: {}'.format(err)) error = True if error is True: - line = input('Do you want to continue? [y/N] ').strip().lower() - if line != 'y': + if not ask_yes_no_question('Do you want to continue?', conf): return False print('<== Done!') return True +def print_help(): + print("generator.py Helper to regenerate gtk-rs crates using gir.") + print("") + print("[OPTIONS]") + print(" -h | --help Display this message") + print(" --gir-path [PATH] Sets the path of the gir executable to run") + print(" (`./gir/target/release/gir` by default)") + print(" --gir-files [PATH] Sets the path of the gir-files folder") + print(" (`gir-files` by default)") + print(" --yes Always answer `yes` to any question asked by the script") + + +def parse_args(args): + conf = { + "gir_path": None, + "gir_files": None, + "yes": False, + } + i = 0 + + while i < len(args): + arg = args[i] + if arg == "-h" or arg == "--help": + print_help() + return None + elif arg == "--gir-path": + i += 1 + if i >= len(args): + print("Expected argument after `--gir-path` option...") + return None + if not isfile(args[i]): + print("`{}` file doesn't exist. Aborting...".format(args[i])) + return None + conf["gir_path"] = args[i] + elif arg == "--gir-files": + i += 1 + if i >= len(args): + print("Expected argument after `--gir-files` option...") + return None + if not isfolder(args[i]): + print("`{}` folder doesn't exist. Aborting...".format(args[i])) + return None + conf["gir_files"] = args[i] + elif arg == "--yes": + conf["yes"] = True + else: + print("Unknown argument `{}`.".format(arg)) + return None + i += 1 + return conf + + def main(): - if def_check_submodule("gir-files") == FAILURE: - return 1 - if not build_gir_if_needed(def_check_submodule("gir")): + gir_path = None + + conf = parse_args(sys.argv[1:]) + if conf is None: return 1 + if conf["gir_files"] is None: + if def_check_submodule("gir-files", conf) == FAILURE: + return 1 + conf["gir_files"] = "gir-files" + if conf["gir_path"] is None: + if not build_gir_if_needed(def_check_submodule("gir", conf)): + return 1 + conf["gir_path"] = "./gir/target/release/gir" + print('=> Regenerating crates...') - if not regen_crates("."): + if not regen_crates(".", conf): return 1 if not run_command(['cargo', 'fmt']): return 1 From d6710a4a4e04a604e7755d7527b3f7d96b54e1da Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 6 Nov 2020 15:37:01 +0100 Subject: [PATCH 04/32] Fix func name --- generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator.py b/generator.py index b5f34a5b3..c1668af81 100755 --- a/generator.py +++ b/generator.py @@ -138,7 +138,7 @@ def parse_args(args): if i >= len(args): print("Expected argument after `--gir-files` option...") return None - if not isfolder(args[i]): + if not isdir(args[i]): print("`{}` folder doesn't exist. Aborting...".format(args[i])) return None conf["gir_files"] = args[i] From ee3bb5600581b19af4224b8bd1b1763576a39727 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 6 Nov 2020 15:47:48 +0100 Subject: [PATCH 05/32] Add option to not run `cargo fmt` --- generator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/generator.py b/generator.py index c1668af81..fcd297a53 100755 --- a/generator.py +++ b/generator.py @@ -109,6 +109,7 @@ def print_help(): print(" --gir-files [PATH] Sets the path of the gir-files folder") print(" (`gir-files` by default)") print(" --yes Always answer `yes` to any question asked by the script") + print(" --no-fmt If set, this script won't run `cargo fmt`") def parse_args(args): @@ -116,6 +117,7 @@ def parse_args(args): "gir_path": None, "gir_files": None, "yes": False, + "run_fmt": True, } i = 0 @@ -144,6 +146,8 @@ def parse_args(args): conf["gir_files"] = args[i] elif arg == "--yes": conf["yes"] = True + elif arg == "--no-fmt": + conf["run_fmt"] = False else: print("Unknown argument `{}`.".format(arg)) return None @@ -170,7 +174,7 @@ def main(): print('=> Regenerating crates...') if not regen_crates(".", conf): return 1 - if not run_command(['cargo', 'fmt']): + if conf["run_fmt"] is True and not run_command(['cargo', 'fmt']): return 1 print('<= Done!') print("Don't forget to check if everything has been correctly generated!") From d47f38752e5272d0066c8c19f7302226e26ec136 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 12 Nov 2020 20:06:22 +0100 Subject: [PATCH 06/32] Run cargo build on gir unless there was a git submodule update failure --- generator.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/generator.py b/generator.py index fcd297a53..a06b8bf6a 100755 --- a/generator.py +++ b/generator.py @@ -63,13 +63,12 @@ def def_check_submodule(submodule_path, conf): def build_gir_if_needed(updated_submodule): if updated_submodule == FAILURE: return False - elif updated_submodule == NEED_UPDATE or not isfile('./gir/target/release/gir'): - print('=> Building gir...') - if update_workspace() is True: - print('<= Done!') - else: - print('<= Failed...') - return False + print('=> Building gir...') + if update_workspace() is True: + print('<= Done!') + else: + print('<= Failed...') + return False return True From fb6a05f8c72dd8f65899698f0528e13947f39a3e Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 17 Nov 2020 21:44:29 +0100 Subject: [PATCH 07/32] Also regen gobject if needed --- generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator.py b/generator.py index a06b8bf6a..29995a247 100755 --- a/generator.py +++ b/generator.py @@ -78,7 +78,7 @@ def regen_crates(path, conf, level=0): if isdir(entry_file): if level < 2 and not regen_crates(entry_file, conf, level + 1): return False - elif entry == 'Gir.toml': + elif entry.startswith("Gir") and entry.endswith(".toml"): print('==> Regenerating "{}"...'.format(entry_file)) args = [conf["gir_path"], '-c', entry_file, '-o', path, '-d', conf["gir_files"]] From 80d148df6b6180c093c7467207e5f7cd2bb65f51 Mon Sep 17 00:00:00 2001 From: Sophie Herold Date: Sun, 22 Nov 2020 14:48:24 +0100 Subject: [PATCH 08/32] [generator.py] Use argparse and option for paths --- generator.py | 103 ++++++++++++++++++--------------------------------- 1 file changed, 37 insertions(+), 66 deletions(-) diff --git a/generator.py b/generator.py index 29995a247..f53ef1893 100755 --- a/generator.py +++ b/generator.py @@ -2,6 +2,7 @@ from os import listdir from os.path import isfile, isdir, join +import argparse import subprocess import sys @@ -10,6 +11,9 @@ NEED_UPDATE = 1 FAILURE = 2 +DEFAULT_GIR_DIRECTORY = 'gir-files' +DEFAULT_GIR_PATH = './gir/target/release/gir' + def run_command(command, folder=None): if folder is None: @@ -31,7 +35,7 @@ def update_workspace(): def ask_yes_no_question(question, conf): question = '{} [y/N] '.format(question) - if conf["yes"] is True: + if conf.yes: print(question) return True if sys.version_info[0] < 3: @@ -64,7 +68,7 @@ def build_gir_if_needed(updated_submodule): if updated_submodule == FAILURE: return False print('=> Building gir...') - if update_workspace() is True: + if update_workspace(): print('<= Done!') else: print('<= Failed...') @@ -81,99 +85,66 @@ def regen_crates(path, conf, level=0): elif entry.startswith("Gir") and entry.endswith(".toml"): print('==> Regenerating "{}"...'.format(entry_file)) - args = [conf["gir_path"], '-c', entry_file, '-o', path, '-d', conf["gir_files"]] + args = [conf.gir_path, '-c', entry_file, '-o', path, '-d', conf.gir_directory] if level > 1: args.append('-m') args.append('sys') error = False try: - error = run_command(args) is False + error = not run_command(args) except Exception as err: print('The following error occurred: {}'.format(err)) error = True - if error is True: + if error: if not ask_yes_no_question('Do you want to continue?', conf): return False print('<== Done!') return True -def print_help(): - print("generator.py Helper to regenerate gtk-rs crates using gir.") - print("") - print("[OPTIONS]") - print(" -h | --help Display this message") - print(" --gir-path [PATH] Sets the path of the gir executable to run") - print(" (`./gir/target/release/gir` by default)") - print(" --gir-files [PATH] Sets the path of the gir-files folder") - print(" (`gir-files` by default)") - print(" --yes Always answer `yes` to any question asked by the script") - print(" --no-fmt If set, this script won't run `cargo fmt`") +def parse_args(args): + parser = argparse.ArgumentParser(description='Helper to regenerate gtk-rs crates using gir.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('path', nargs="*", default='.', + help='Paths in which to look for Gir.toml files') + parser.add_argument('--gir-directory', default=DEFAULT_GIR_DIRECTORY, + help='Path of the gir-files folder') + parser.add_argument('--gir-path', default=DEFAULT_GIR_PATH, + help='Path of the gir executable to run') + parser.add_argument('--yes', action='store_true', + help=' Always answer `yes` to any question asked by the script') + parser.add_argument('--no-fmt', action='store_true', + help='If set, this script will not run `cargo fmt`') -def parse_args(args): - conf = { - "gir_path": None, - "gir_files": None, - "yes": False, - "run_fmt": True, - } - i = 0 - - while i < len(args): - arg = args[i] - if arg == "-h" or arg == "--help": - print_help() - return None - elif arg == "--gir-path": - i += 1 - if i >= len(args): - print("Expected argument after `--gir-path` option...") - return None - if not isfile(args[i]): - print("`{}` file doesn't exist. Aborting...".format(args[i])) - return None - conf["gir_path"] = args[i] - elif arg == "--gir-files": - i += 1 - if i >= len(args): - print("Expected argument after `--gir-files` option...") - return None - if not isdir(args[i]): - print("`{}` folder doesn't exist. Aborting...".format(args[i])) - return None - conf["gir_files"] = args[i] - elif arg == "--yes": - conf["yes"] = True - elif arg == "--no-fmt": - conf["run_fmt"] = False - else: - print("Unknown argument `{}`.".format(arg)) - return None - i += 1 - return conf + return parser.parse_args() def main(): gir_path = None conf = parse_args(sys.argv[1:]) - if conf is None: - return 1 - if conf["gir_files"] is None: + if conf.gir_directory == DEFAULT_GIR_DIRECTORY: if def_check_submodule("gir-files", conf) == FAILURE: return 1 - conf["gir_files"] = "gir-files" - if conf["gir_path"] is None: + elif not isdir(conf.gir_directory): + print("`{}` dir doesn't exist. Aborting...".format(path)) + return 1 + + if conf.gir_path == DEFAULT_GIR_PATH: if not build_gir_if_needed(def_check_submodule("gir", conf)): return 1 - conf["gir_path"] = "./gir/target/release/gir" + elif not isfile(conf.gir_path): + print("`{}` file doesn't exist. Aborting...".format(path)) + return 1 print('=> Regenerating crates...') - if not regen_crates(".", conf): - return 1 - if conf["run_fmt"] is True and not run_command(['cargo', 'fmt']): + for path in conf.path: + print('=> Looking in path `{}`'.format(path)) + if not regen_crates(path, conf): + return 1 + if not conf.no_fmt and not run_command(['cargo', 'fmt']): return 1 print('<= Done!') print("Don't forget to check if everything has been correctly generated!") From e242dd991b99c3d18c83a86f5c29063517015ece Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 22 Nov 2020 16:20:44 +0100 Subject: [PATCH 09/32] generator: Fix "`path` is not defined" warning --- generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generator.py b/generator.py index f53ef1893..f7c01185b 100755 --- a/generator.py +++ b/generator.py @@ -129,14 +129,14 @@ def main(): if def_check_submodule("gir-files", conf) == FAILURE: return 1 elif not isdir(conf.gir_directory): - print("`{}` dir doesn't exist. Aborting...".format(path)) + print("`{}` dir doesn't exist. Aborting...".format(conf.gir_directory)) return 1 if conf.gir_path == DEFAULT_GIR_PATH: if not build_gir_if_needed(def_check_submodule("gir", conf)): return 1 elif not isfile(conf.gir_path): - print("`{}` file doesn't exist. Aborting...".format(path)) + print("`{}` file doesn't exist. Aborting...".format(conf.gir_path)) return 1 print('=> Regenerating crates...') From b83289b358113abc9b9da045ff9d68b9511911bc Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 22 Nov 2020 16:53:38 +0100 Subject: [PATCH 10/32] generator: Use requested path in def_check_submodule --- generator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/generator.py b/generator.py index f7c01185b..c346ca3de 100755 --- a/generator.py +++ b/generator.py @@ -48,16 +48,16 @@ def ask_yes_no_question(question, conf): def def_check_submodule(submodule_path, conf): if len(listdir(submodule_path)) != 0: return NOTHING_TO_BE_DONE - print('=> Initializing gir submodule...') - if not run_command(['git', 'submodule', 'update', '--init']): + print('=> Initializing {} submodule...'.format(submodule_path)) + if not run_command(['git', 'submodule', 'update', '--init', submodule_path]): return FAILURE print('<= Done!') - if ask_yes_no_question('Do you want to update gir submodule?', conf): - print('=> Updating gir submodule...') - if not run_command(['git', 'reset', '--hard', 'HEAD'], 'gir'): + if ask_yes_no_question('Do you want to update {} submodule?'.format(submodule_path), conf): + print('=> Updating submodule...') + if not run_command(['git', 'reset', '--hard', 'HEAD'], submodule_path): return FAILURE - if not run_command(['git', 'pull', '-f', 'origin', 'master'], 'gir'): + if not run_command(['git', 'pull', '-f', 'origin', 'master'], submodule_path): return FAILURE print('<= Done!') return NEED_UPDATE From 12eb1511f1857aec5526ce5275e4e196ea521151 Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 22 Nov 2020 17:00:44 +0100 Subject: [PATCH 11/32] generator: Rename --gir-directory to --gir-files gir-directory mentally conflicts with the directory for `gir`, rather than the specifically-named gir-files directory. --- generator.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/generator.py b/generator.py index c346ca3de..ed62d83e4 100755 --- a/generator.py +++ b/generator.py @@ -11,8 +11,9 @@ NEED_UPDATE = 1 FAILURE = 2 -DEFAULT_GIR_DIRECTORY = 'gir-files' -DEFAULT_GIR_PATH = './gir/target/release/gir' +DEFAULT_GIR_FILES_DIRECTORY = './gir-files' +DEFAULT_GIR_DIRECTORY = './gir/' +DEFAULT_GIR_PATH = DEFAULT_GIR_DIRECTORY + 'target/release/gir' def run_command(command, folder=None): @@ -85,7 +86,7 @@ def regen_crates(path, conf, level=0): elif entry.startswith("Gir") and entry.endswith(".toml"): print('==> Regenerating "{}"...'.format(entry_file)) - args = [conf.gir_path, '-c', entry_file, '-o', path, '-d', conf.gir_directory] + args = [conf.gir_path, '-c', entry_file, '-o', path, '-d', conf.gir_files_path] if level > 1: args.append('-m') args.append('sys') @@ -102,13 +103,13 @@ def regen_crates(path, conf, level=0): return True -def parse_args(args): +def parse_args(): parser = argparse.ArgumentParser(description='Helper to regenerate gtk-rs crates using gir.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('path', nargs="*", default='.', help='Paths in which to look for Gir.toml files') - parser.add_argument('--gir-directory', default=DEFAULT_GIR_DIRECTORY, + parser.add_argument('--gir-files', dest="gir_files_path", default=DEFAULT_GIR_FILES_DIRECTORY, help='Path of the gir-files folder') parser.add_argument('--gir-path', default=DEFAULT_GIR_PATH, help='Path of the gir executable to run') @@ -121,19 +122,17 @@ def parse_args(args): def main(): - gir_path = None + conf = parse_args() - conf = parse_args(sys.argv[1:]) - - if conf.gir_directory == DEFAULT_GIR_DIRECTORY: - if def_check_submodule("gir-files", conf) == FAILURE: + if conf.gir_files_path == DEFAULT_GIR_FILES_DIRECTORY: + if def_check_submodule(conf.gir_files_path, conf) == FAILURE: return 1 - elif not isdir(conf.gir_directory): - print("`{}` dir doesn't exist. Aborting...".format(conf.gir_directory)) + elif not isdir(conf.gir_files_path): + print("`{}` dir doesn't exist. Aborting...".format(conf.gir_files_path)) return 1 if conf.gir_path == DEFAULT_GIR_PATH: - if not build_gir_if_needed(def_check_submodule("gir", conf)): + if not build_gir_if_needed(def_check_submodule(DEFAULT_GIR_DIRECTORY, conf)): return 1 elif not isfile(conf.gir_path): print("`{}` file doesn't exist. Aborting...".format(conf.gir_path)) From 509c54db7bf2e20627652d2d1042dc23aedbe2bb Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 22 Nov 2020 17:07:23 +0100 Subject: [PATCH 12/32] generator: Convert os.path handling to pathlib, initialized in argparse --- generator.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/generator.py b/generator.py index ed62d83e4..9017d8fe1 100755 --- a/generator.py +++ b/generator.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -from os import listdir -from os.path import isfile, isdir, join +from pathlib import Path import argparse import subprocess import sys @@ -11,9 +10,9 @@ NEED_UPDATE = 1 FAILURE = 2 -DEFAULT_GIR_FILES_DIRECTORY = './gir-files' -DEFAULT_GIR_DIRECTORY = './gir/' -DEFAULT_GIR_PATH = DEFAULT_GIR_DIRECTORY + 'target/release/gir' +DEFAULT_GIR_FILES_DIRECTORY = Path('./gir-files') +DEFAULT_GIR_DIRECTORY = Path('./gir/') +DEFAULT_GIR_PATH = DEFAULT_GIR_DIRECTORY / 'target/release/gir' def run_command(command, folder=None): @@ -37,7 +36,7 @@ def update_workspace(): def ask_yes_no_question(question, conf): question = '{} [y/N] '.format(question) if conf.yes: - print(question) + print(question + 'y') return True if sys.version_info[0] < 3: line = raw_input(question) @@ -47,7 +46,7 @@ def ask_yes_no_question(question, conf): def def_check_submodule(submodule_path, conf): - if len(listdir(submodule_path)) != 0: + if any(submodule_path.iterdir()): return NOTHING_TO_BE_DONE print('=> Initializing {} submodule...'.format(submodule_path)) if not run_command(['git', 'submodule', 'update', '--init', submodule_path]): @@ -78,18 +77,16 @@ def build_gir_if_needed(updated_submodule): def regen_crates(path, conf, level=0): - for entry in listdir(path): - entry_file = join(path, entry) - if isdir(entry_file): - if level < 2 and not regen_crates(entry_file, conf, level + 1): + for entry in path.iterdir(): + if entry.is_dir(): + if level < 2 and not regen_crates(entry, conf, level + 1): return False - elif entry.startswith("Gir") and entry.endswith(".toml"): - print('==> Regenerating "{}"...'.format(entry_file)) + elif entry.name.startswith("Gir") and entry.suffix == ".toml": + print('==> Regenerating "{}"...'.format(entry)) - args = [conf.gir_path, '-c', entry_file, '-o', path, '-d', conf.gir_files_path] + args = [conf.gir_path, '-c', entry, '-o', entry.parent, '-d', conf.gir_files_path] if level > 1: - args.append('-m') - args.append('sys') + args.extend(['-m', 'sys']) error = False try: error = not run_command(args) @@ -107,11 +104,14 @@ def parse_args(): parser = argparse.ArgumentParser(description='Helper to regenerate gtk-rs crates using gir.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('path', nargs="*", default='.', + parser.add_argument('path', nargs="*", default=[Path('.')], + type=Path, help='Paths in which to look for Gir.toml files') parser.add_argument('--gir-files', dest="gir_files_path", default=DEFAULT_GIR_FILES_DIRECTORY, + type=Path, help='Path of the gir-files folder') parser.add_argument('--gir-path', default=DEFAULT_GIR_PATH, + type=Path, help='Path of the gir executable to run') parser.add_argument('--yes', action='store_true', help=' Always answer `yes` to any question asked by the script') @@ -127,14 +127,14 @@ def main(): if conf.gir_files_path == DEFAULT_GIR_FILES_DIRECTORY: if def_check_submodule(conf.gir_files_path, conf) == FAILURE: return 1 - elif not isdir(conf.gir_files_path): + elif not conf.gir_files_path.is_dir(): print("`{}` dir doesn't exist. Aborting...".format(conf.gir_files_path)) return 1 if conf.gir_path == DEFAULT_GIR_PATH: if not build_gir_if_needed(def_check_submodule(DEFAULT_GIR_DIRECTORY, conf)): return 1 - elif not isfile(conf.gir_path): + elif not conf.gir_path.is_file(): print("`{}` file doesn't exist. Aborting...".format(conf.gir_path)) return 1 From ecfbf13b4a1240b360789c500e76ed5389030b6d Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 22 Nov 2020 17:09:58 +0100 Subject: [PATCH 13/32] generator: Immediately check for valid file/directory in argparse Argparse conveniently prints the name of the argument that failed parsing. --- generator.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/generator.py b/generator.py index 9017d8fe1..4b2c18d1c 100755 --- a/generator.py +++ b/generator.py @@ -100,18 +100,32 @@ def regen_crates(path, conf, level=0): return True +def directory_path(path): + path = Path(path) + if not path.is_dir(): + raise argparse.ArgumentTypeError("`{}` directory not found".format(path)) + return path + + +def file_path(path): + path = Path(path) + if not path.is_file(): + raise argparse.ArgumentTypeError("`{}` file not found".format(path)) + return path + + def parse_args(): parser = argparse.ArgumentParser(description='Helper to regenerate gtk-rs crates using gir.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('path', nargs="*", default=[Path('.')], - type=Path, + type=directory_path, help='Paths in which to look for Gir.toml files') parser.add_argument('--gir-files', dest="gir_files_path", default=DEFAULT_GIR_FILES_DIRECTORY, - type=Path, + type=directory_path, help='Path of the gir-files folder') parser.add_argument('--gir-path', default=DEFAULT_GIR_PATH, - type=Path, + type=file_path, help='Path of the gir executable to run') parser.add_argument('--yes', action='store_true', help=' Always answer `yes` to any question asked by the script') @@ -127,16 +141,10 @@ def main(): if conf.gir_files_path == DEFAULT_GIR_FILES_DIRECTORY: if def_check_submodule(conf.gir_files_path, conf) == FAILURE: return 1 - elif not conf.gir_files_path.is_dir(): - print("`{}` dir doesn't exist. Aborting...".format(conf.gir_files_path)) - return 1 if conf.gir_path == DEFAULT_GIR_PATH: if not build_gir_if_needed(def_check_submodule(DEFAULT_GIR_DIRECTORY, conf)): return 1 - elif not conf.gir_path.is_file(): - print("`{}` file doesn't exist. Aborting...".format(conf.gir_path)) - return 1 print('=> Regenerating crates...') for path in conf.path: From 6b31dede5e5b52e074ad19c6d0e8144d029d5b78 Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 22 Nov 2020 17:14:58 +0100 Subject: [PATCH 14/32] generator: Don't guess sys crate based on level but name Now that paths can be specified on the commandline the depth level does not need to represent sys crates anymore. --- generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator.py b/generator.py index 4b2c18d1c..91e55d274 100755 --- a/generator.py +++ b/generator.py @@ -85,7 +85,7 @@ def regen_crates(path, conf, level=0): print('==> Regenerating "{}"...'.format(entry)) args = [conf.gir_path, '-c', entry, '-o', entry.parent, '-d', conf.gir_files_path] - if level > 1: + if entry.parent.name.endswith("sys"): args.extend(['-m', 'sys']) error = False try: From 3a366fb8553f203c17ced8c6298b4a9c86ac282b Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 22 Nov 2020 17:23:57 +0100 Subject: [PATCH 15/32] generator: Convert Popen to run --- generator.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/generator.py b/generator.py index 91e55d274..820edaca4 100755 --- a/generator.py +++ b/generator.py @@ -18,19 +18,15 @@ def run_command(command, folder=None): if folder is None: folder = "." - child = subprocess.Popen(command, cwd=folder) - child.communicate() - if child.returncode != 0: - print("Command `{}` failed with return code `{}`...".format(command, child.returncode)) + ret = subprocess.run(command, cwd=folder) + if ret.returncode != 0: + print("Command `{}` failed with `{}`...".format(command, ret)) return False return True def update_workspace(): - try: - return run_command(['cargo', 'build', '--release'], 'gir') - except: - return False + return run_command(['cargo', 'build', '--release'], 'gir') def ask_yes_no_question(question, conf): From 8bbdea39c9b08caeec8b39058eb2615f21899ec0 Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 22 Nov 2020 21:26:43 +0100 Subject: [PATCH 16/32] generator: Allow passing paths to files and directories --- generator.py | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/generator.py b/generator.py index 820edaca4..36cc80a3c 100755 --- a/generator.py +++ b/generator.py @@ -73,29 +73,36 @@ def build_gir_if_needed(updated_submodule): def regen_crates(path, conf, level=0): - for entry in path.iterdir(): - if entry.is_dir(): - if level < 2 and not regen_crates(entry, conf, level + 1): + if path.is_dir(): + for entry in path.iterdir(): + if level < 3 and not regen_crates(entry, conf, level + 1): return False - elif entry.name.startswith("Gir") and entry.suffix == ".toml": - print('==> Regenerating "{}"...'.format(entry)) - - args = [conf.gir_path, '-c', entry, '-o', entry.parent, '-d', conf.gir_files_path] - if entry.parent.name.endswith("sys"): - args.extend(['-m', 'sys']) - error = False - try: - error = not run_command(args) - except Exception as err: - print('The following error occurred: {}'.format(err)) - error = True - if error: - if not ask_yes_no_question('Do you want to continue?', conf): - return False - print('<== Done!') + elif path.name.startswith("Gir") and path.suffix == ".toml": + print('==> Regenerating "{}"...'.format(path)) + + args = [conf.gir_path, '-c', path, '-o', path.parent, '-d', conf.gir_files_path] + if path.parent.name.endswith("sys"): + args.extend(['-m', 'sys']) + error = False + try: + error = not run_command(args) + except Exception as err: + print('The following error occurred: {}'.format(err)) + error = True + if error: + if not ask_yes_no_question('Do you want to continue?', conf): + return False + print('<== Done!') return True +def valid_path(path): + path = Path(path) + if not path.exists(): + raise argparse.ArgumentTypeError("`{}` no such file or directory".format(path)) + return path + + def directory_path(path): path = Path(path) if not path.is_dir(): @@ -115,7 +122,7 @@ def parse_args(): formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('path', nargs="*", default=[Path('.')], - type=directory_path, + type=valid_path, help='Paths in which to look for Gir.toml files') parser.add_argument('--gir-files', dest="gir_files_path", default=DEFAULT_GIR_FILES_DIRECTORY, type=directory_path, From b1345c19e988ba79ded89ecd7779d060b46abfbf Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 22 Nov 2020 21:31:38 +0100 Subject: [PATCH 17/32] generator: Use pathlibs rglob to find all Gir*.toml --- generator.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/generator.py b/generator.py index 36cc80a3c..07ee29090 100755 --- a/generator.py +++ b/generator.py @@ -72,12 +72,12 @@ def build_gir_if_needed(updated_submodule): return True -def regen_crates(path, conf, level=0): +def regen_crates(path, conf): if path.is_dir(): - for entry in path.iterdir(): - if level < 3 and not regen_crates(entry, conf, level + 1): + for entry in path.rglob("Gir*.toml"): + if not regen_crates(entry, conf): return False - elif path.name.startswith("Gir") and path.suffix == ".toml": + elif path.match("Gir*.toml"): print('==> Regenerating "{}"...'.format(path)) args = [conf.gir_path, '-c', path, '-o', path.parent, '-d', conf.gir_files_path] @@ -93,6 +93,9 @@ def regen_crates(path, conf, level=0): if not ask_yes_no_question('Do you want to continue?', conf): return False print('<== Done!') + else: + print('==> {} is not a valid Gir*.toml file'.format(path)) + return False return True From 18a03923de8b8991cdf424fa7c11cea491c48bdf Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 23 Nov 2020 21:18:36 +0100 Subject: [PATCH 18/32] Improve naming --- generator.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/generator.py b/generator.py index f53ef1893..7d8afb57f 100755 --- a/generator.py +++ b/generator.py @@ -11,7 +11,7 @@ NEED_UPDATE = 1 FAILURE = 2 -DEFAULT_GIR_DIRECTORY = 'gir-files' +DEFAULT_GIR_FILES_DIRECTORY = 'gir-files' DEFAULT_GIR_PATH = './gir/target/release/gir' @@ -85,7 +85,7 @@ def regen_crates(path, conf, level=0): elif entry.startswith("Gir") and entry.endswith(".toml"): print('==> Regenerating "{}"...'.format(entry_file)) - args = [conf.gir_path, '-c', entry_file, '-o', path, '-d', conf.gir_directory] + args = [conf.gir_path, '-c', entry_file, '-o', path, '-d', conf.gir_files_directory] if level > 1: args.append('-m') args.append('sys') @@ -108,7 +108,7 @@ def parse_args(args): parser.add_argument('path', nargs="*", default='.', help='Paths in which to look for Gir.toml files') - parser.add_argument('--gir-directory', default=DEFAULT_GIR_DIRECTORY, + parser.add_argument('--gir-files-directory', default=DEFAULT_GIR_FILES_DIRECTORY, help='Path of the gir-files folder') parser.add_argument('--gir-path', default=DEFAULT_GIR_PATH, help='Path of the gir executable to run') @@ -125,11 +125,11 @@ def main(): conf = parse_args(sys.argv[1:]) - if conf.gir_directory == DEFAULT_GIR_DIRECTORY: - if def_check_submodule("gir-files", conf) == FAILURE: + if conf.gir_files_directory == DEFAULT_GIR_FILES_DIRECTORY: + if def_check_submodule(DEFAULT_GIR_FILES_DIRECTORY, conf) == FAILURE: return 1 - elif not isdir(conf.gir_directory): - print("`{}` dir doesn't exist. Aborting...".format(path)) + elif not isdir(conf.gir_files_directory): + print("`{}` dir doesn't exist. Aborting...".format(conf.gir_files_directory)) return 1 if conf.gir_path == DEFAULT_GIR_PATH: From 0b79fd0f08b0a2b4abb4700097127f33a45fcea9 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 24 Nov 2020 11:47:11 +0100 Subject: [PATCH 19/32] Remove useless check --- generator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/generator.py b/generator.py index 9b3dce1fc..3e4dd20c0 100755 --- a/generator.py +++ b/generator.py @@ -147,9 +147,6 @@ def main(): if conf.gir_files_path == DEFAULT_GIR_FILES_DIRECTORY: if def_check_submodule(conf.gir_files_path, conf) == FAILURE: return 1 - elif not isdir(conf.gir_files_path): - print("`{}` dir doesn't exist. Aborting...".format(conf.gir_files_path)) - return 1 if conf.gir_path == DEFAULT_GIR_PATH: if not build_gir_if_needed(def_check_submodule(DEFAULT_GIR_DIRECTORY, conf)): From 5739c3f3f106c5d895b7fe69671d3871309edeb9 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 24 Nov 2020 14:43:47 +0100 Subject: [PATCH 20/32] Remove python2 code --- generator.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/generator.py b/generator.py index 3e4dd20c0..cf20407fe 100755 --- a/generator.py +++ b/generator.py @@ -34,10 +34,7 @@ def ask_yes_no_question(question, conf): if conf.yes: print(question + 'y') return True - if sys.version_info[0] < 3: - line = raw_input(question) - else: - line = input(question) + line = input(question) return line.strip().lower() == 'y' From af0ae9d71c69831b069d2b1d48db826a7c947832 Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Tue, 24 Nov 2020 20:20:57 +0100 Subject: [PATCH 21/32] generator: Autoformat and normalize quotation with `black` --- generator.py | 109 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 44 deletions(-) diff --git a/generator.py b/generator.py index cf20407fe..3798bfb4c 100755 --- a/generator.py +++ b/generator.py @@ -10,9 +10,9 @@ NEED_UPDATE = 1 FAILURE = 2 -DEFAULT_GIR_FILES_DIRECTORY = Path('./gir-files') -DEFAULT_GIR_DIRECTORY = Path('./gir/') -DEFAULT_GIR_PATH = DEFAULT_GIR_DIRECTORY / 'target/release/gir' +DEFAULT_GIR_FILES_DIRECTORY = Path("./gir-files") +DEFAULT_GIR_DIRECTORY = Path("./gir/") +DEFAULT_GIR_PATH = DEFAULT_GIR_DIRECTORY / "target/release/gir" def run_command(command, folder=None): @@ -26,33 +26,35 @@ def run_command(command, folder=None): def update_workspace(): - return run_command(['cargo', 'build', '--release'], 'gir') + return run_command(["cargo", "build", "--release"], "gir") def ask_yes_no_question(question, conf): - question = '{} [y/N] '.format(question) + question = "{} [y/N] ".format(question) if conf.yes: - print(question + 'y') + print(question + "y") return True line = input(question) - return line.strip().lower() == 'y' + return line.strip().lower() == "y" def def_check_submodule(submodule_path, conf): if any(submodule_path.iterdir()): return NOTHING_TO_BE_DONE - print('=> Initializing {} submodule...'.format(submodule_path)) - if not run_command(['git', 'submodule', 'update', '--init', submodule_path]): + print("=> Initializing {} submodule...".format(submodule_path)) + if not run_command(["git", "submodule", "update", "--init", submodule_path]): return FAILURE - print('<= Done!') + print("<= Done!") - if ask_yes_no_question('Do you want to update {} submodule?'.format(submodule_path), conf): - print('=> Updating submodule...') - if not run_command(['git', 'reset', '--hard', 'HEAD'], submodule_path): + if ask_yes_no_question( + "Do you want to update {} submodule?".format(submodule_path), conf + ): + print("=> Updating submodule...") + if not run_command(["git", "reset", "--hard", "HEAD"], submodule_path): return FAILURE - if not run_command(['git', 'pull', '-f', 'origin', 'master'], submodule_path): + if not run_command(["git", "pull", "-f", "origin", "master"], submodule_path): return FAILURE - print('<= Done!') + print("<= Done!") return NEED_UPDATE return NOTHING_TO_BE_DONE @@ -60,11 +62,11 @@ def def_check_submodule(submodule_path, conf): def build_gir_if_needed(updated_submodule): if updated_submodule == FAILURE: return False - print('=> Building gir...') + print("=> Building gir...") if update_workspace(): - print('<= Done!') + print("<= Done!") else: - print('<= Failed...') + print("<= Failed...") return False return True @@ -77,21 +79,21 @@ def regen_crates(path, conf): elif path.match("Gir*.toml"): print('==> Regenerating "{}"...'.format(path)) - args = [conf.gir_path, '-c', path, '-o', path.parent, '-d', conf.gir_files_path] + args = [conf.gir_path, "-c", path, "-o", path.parent, "-d", conf.gir_files_path] if path.parent.name.endswith("sys"): - args.extend(['-m', 'sys']) + args.extend(["-m", "sys"]) error = False try: error = not run_command(args) except Exception as err: - print('The following error occurred: {}'.format(err)) + print("The following error occurred: {}".format(err)) error = True if error: - if not ask_yes_no_question('Do you want to continue?', conf): + if not ask_yes_no_question("Do you want to continue?", conf): return False - print('<== Done!') + print("<== Done!") else: - print('==> {} is not a valid Gir*.toml file'.format(path)) + print("==> {} is not a valid Gir*.toml file".format(path)) return False return True @@ -118,22 +120,41 @@ def file_path(path): def parse_args(): - parser = argparse.ArgumentParser(description='Helper to regenerate gtk-rs crates using gir.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('path', nargs="*", default=[Path('.')], - type=valid_path, - help='Paths in which to look for Gir.toml files') - parser.add_argument('--gir-files-directory', dest="gir_files_path", default=DEFAULT_GIR_FILES_DIRECTORY, - type=directory_path, - help='Path of the gir-files folder') - parser.add_argument('--gir-path', default=DEFAULT_GIR_PATH, - type=file_path, - help='Path of the gir executable to run') - parser.add_argument('--yes', action='store_true', - help=' Always answer `yes` to any question asked by the script') - parser.add_argument('--no-fmt', action='store_true', - help='If set, this script will not run `cargo fmt`') + parser = argparse.ArgumentParser( + description="Helper to regenerate gtk-rs crates using gir.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "path", + nargs="*", + default=[Path(".")], + type=valid_path, + help="Paths in which to look for Gir.toml files", + ) + parser.add_argument( + "--gir-files-directory", + dest="gir_files_path", + default=DEFAULT_GIR_FILES_DIRECTORY, + type=directory_path, + help="Path of the gir-files folder", + ) + parser.add_argument( + "--gir-path", + default=DEFAULT_GIR_PATH, + type=file_path, + help="Path of the gir executable to run", + ) + parser.add_argument( + "--yes", + action="store_true", + help=" Always answer `yes` to any question asked by the script", + ) + parser.add_argument( + "--no-fmt", + action="store_true", + help="If set, this script will not run `cargo fmt`", + ) return parser.parse_args() @@ -149,14 +170,14 @@ def main(): if not build_gir_if_needed(def_check_submodule(DEFAULT_GIR_DIRECTORY, conf)): return 1 - print('=> Regenerating crates...') + print("=> Regenerating crates...") for path in conf.path: - print('=> Looking in path `{}`'.format(path)) + print("=> Looking in path `{}`".format(path)) if not regen_crates(path, conf): return 1 - if not conf.no_fmt and not run_command(['cargo', 'fmt']): + if not conf.no_fmt and not run_command(["cargo", "fmt"]): return 1 - print('<= Done!') + print("<= Done!") print("Don't forget to check if everything has been correctly generated!") return 0 From 132a1c9a38f1f5d53214c231ff04ba3446f43f46 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 19 Mar 2021 22:13:02 +0100 Subject: [PATCH 22/32] Don't set a default value for the -d option --- generator.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/generator.py b/generator.py index 3798bfb4c..2b45f8651 100755 --- a/generator.py +++ b/generator.py @@ -79,7 +79,9 @@ def regen_crates(path, conf): elif path.match("Gir*.toml"): print('==> Regenerating "{}"...'.format(path)) - args = [conf.gir_path, "-c", path, "-o", path.parent, "-d", conf.gir_files_path] + args = [conf.gir_path, "-c", path, "-o", path.parent] + if len(conf.gir_files_paths) > 0: + args += ["-d"] + [conf.gir_files_paths] if path.parent.name.endswith("sys"): args.extend(["-m", "sys"]) error = False @@ -133,9 +135,10 @@ def parse_args(): help="Paths in which to look for Gir.toml files", ) parser.add_argument( - "--gir-files-directory", - dest="gir_files_path", - default=DEFAULT_GIR_FILES_DIRECTORY, + "--gir-files-directories", + nargs='+', # If the option is used, we expect at least one folder! + dest="gir_files_paths", + default=[], type=directory_path, help="Path of the gir-files folder", ) @@ -162,8 +165,8 @@ def parse_args(): def main(): conf = parse_args() - if conf.gir_files_path == DEFAULT_GIR_FILES_DIRECTORY: - if def_check_submodule(conf.gir_files_path, conf) == FAILURE: + if len(conf.gir_files_paths) == 0: + if def_check_submodule(DEFAULT_GIR_FILES_DIRECTORY, conf) == FAILURE: return 1 if conf.gir_path == DEFAULT_GIR_PATH: From d6589441d396ae5603bfe05af3f21badd011519e Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Wed, 24 Mar 2021 21:10:04 +0100 Subject: [PATCH 23/32] generator: Iterate gir directories passed on the command line This change cleans up handling of the default gir directory, and makes sure the `-d` flag is prepended to every path. Before the command would contain a single `-d` flag followed by a `list` of paths, which cannot be passed to an executable verbatim: The following error occurred: expected str, bytes or os.PathLike object, not list Do you want to continue? [y/N] ^CTraceback (most recent call last): File "./generator.py", line 189, in Fixes: c33495c ("Don't set a default value for the -d option") --- generator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/generator.py b/generator.py index 2b45f8651..b3232e014 100755 --- a/generator.py +++ b/generator.py @@ -79,9 +79,9 @@ def regen_crates(path, conf): elif path.match("Gir*.toml"): print('==> Regenerating "{}"...'.format(path)) - args = [conf.gir_path, "-c", path, "-o", path.parent] - if len(conf.gir_files_paths) > 0: - args += ["-d"] + [conf.gir_files_paths] + args = [conf.gir_path, "-c", path, "-o", path.parent] + [ + d for path in conf.gir_files_paths for d in ("-d", path) + ] if path.parent.name.endswith("sys"): args.extend(["-m", "sys"]) error = False @@ -136,7 +136,7 @@ def parse_args(): ) parser.add_argument( "--gir-files-directories", - nargs='+', # If the option is used, we expect at least one folder! + nargs="+", # If the option is used, we expect at least one folder! dest="gir_files_paths", default=[], type=directory_path, @@ -165,7 +165,7 @@ def parse_args(): def main(): conf = parse_args() - if len(conf.gir_files_paths) == 0: + if not conf.gir_files_paths: if def_check_submodule(DEFAULT_GIR_FILES_DIRECTORY, conf) == FAILURE: return 1 From 5547e5ad4ae15162e4bb75b44a6597269f682b50 Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 11 Apr 2021 15:44:50 +0200 Subject: [PATCH 24/32] generator: Add --with-docs to run `gir -m doc` for non-sys crates Convenience method to emit documentation for all crates with a `Gir.toml` file. Optionally takes a directory argument for GStreamer-rs, where all documentation ends up in the `docs/` folder in the root rather than in the respective crate folders. --- generator.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/generator.py b/generator.py index b3232e014..ed1ea3d82 100755 --- a/generator.py +++ b/generator.py @@ -77,16 +77,34 @@ def regen_crates(path, conf): if not regen_crates(entry, conf): return False elif path.match("Gir*.toml"): - print('==> Regenerating "{}"...'.format(path)) - args = [conf.gir_path, "-c", path, "-o", path.parent] + [ d for path in conf.gir_files_paths for d in ("-d", path) ] + error = False if path.parent.name.endswith("sys"): args.extend(["-m", "sys"]) - error = False + elif conf.with_docs: + # Update docs/**/docs.md for non-sys crates + + doc_path = "docs.md" + if isinstance(conf.with_docs, Path): + # doc-target-path is relative to `-c` + path_depth = len(path.parent.parts) + doc_path = Path( + *[".."] * path_depth, conf.with_docs, path.parent, doc_path + ) + print( + "==> Regenerating documentation for `{}` into `{}`...".format( + path, doc_path + ) + ) + doc_args = args + ["-m", "doc", "--doc-target-path", doc_path] + error |= not run_command(doc_args) + + print('==> Regenerating "{}"...'.format(path)) + try: - error = not run_command(args) + error |= not run_command(args) except Exception as err: print("The following error occurred: {}".format(err)) error = True @@ -158,6 +176,15 @@ def parse_args(): action="store_true", help="If set, this script will not run `cargo fmt`", ) + parser.add_argument( + "--with-docs", + metavar="output_path", + nargs="?", + const=True, + default=False, + type=directory_path, + help="Build documentation with `gir -m doc`. Optionally takes an output directory", + ) return parser.parse_args() From 1d7e839f7cb65d90e8e10eaa87773b682372ad4d Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 11 Apr 2021 19:52:22 +0200 Subject: [PATCH 25/32] generator: Run all gir processes in parallel Since the addition of doc regeneration - which also spawns a gir process for every non-sys crate - the process is now incredibly slow and not well suited for iterative development: ./generator.py --no-fmt 26.25s user 0.79s system 99% cpu 27.044 total All gir processes are currently ran in serial (the generator waits for one to complete before spawning the next process) even though there are no inter-dependencies. Simply spawning all processes at once and collecting their results + printing them in order after everything has been spawned yields a significant speedup: ./generator.py --no-fmt 37.99s user 0.88s system 3285% cpu 1.183 total Note: this is on a 32-core ThreadRipper. The improvement is more modest on machines with less cores, and also depends on IO speed. A 4-core i5, before and after: ./generator.py --no-fmt 30.24s user 0.76s system 99% cpu 31.055 total ./generator.py --no-fmt 57.78s user 0.88s system 763% cpu 7.685 total That's still a sizable gain for simply not blocking on other tasks anymore. --- generator.py | 54 ++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/generator.py b/generator.py index ed1ea3d82..a25710991 100755 --- a/generator.py +++ b/generator.py @@ -25,6 +25,10 @@ def run_command(command, folder=None): return True +def spawn_process(command): + return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def update_workspace(): return run_command(["cargo", "build", "--release"], "gir") @@ -72,15 +76,15 @@ def build_gir_if_needed(updated_submodule): def regen_crates(path, conf): + processes = [] if path.is_dir(): for entry in path.rglob("Gir*.toml"): - if not regen_crates(entry, conf): - return False + processes += regen_crates(entry, conf) elif path.match("Gir*.toml"): args = [conf.gir_path, "-c", path, "-o", path.parent] + [ d for path in conf.gir_files_paths for d in ("-d", path) ] - error = False + if path.parent.name.endswith("sys"): args.extend(["-m", "sys"]) elif conf.with_docs: @@ -93,29 +97,22 @@ def regen_crates(path, conf): doc_path = Path( *[".."] * path_depth, conf.with_docs, path.parent, doc_path ) - print( - "==> Regenerating documentation for `{}` into `{}`...".format( - path, doc_path + doc_args = args + ["-m", "doc", "--doc-target-path", doc_path] + processes.append( + ( + "Regenerating documentation for `{}` into `{}`...".format( + path, doc_path + ), + spawn_process(doc_args), ) ) - doc_args = args + ["-m", "doc", "--doc-target-path", doc_path] - error |= not run_command(doc_args) - - print('==> Regenerating "{}"...'.format(path)) - - try: - error |= not run_command(args) - except Exception as err: - print("The following error occurred: {}".format(err)) - error = True - if error: - if not ask_yes_no_question("Do you want to continue?", conf): - return False - print("<== Done!") + + processes.append(("Regenerating `{}`...".format(path), spawn_process(args))) + else: - print("==> {} is not a valid Gir*.toml file".format(path)) - return False - return True + raise Exception("`{}` is not a valid Gir*.toml file".format(path)) + + return processes def valid_path(path): @@ -203,8 +200,15 @@ def main(): print("=> Regenerating crates...") for path in conf.path: print("=> Looking in path `{}`".format(path)) - if not regen_crates(path, conf): - return 1 + processes = regen_crates(path, conf) + for log, p in processes: + print("==> {}".format(log)) + stdout, stderr = p.communicate() + # Gir doesn't print anything to stdout. If it does, this is likely out of + # order with stderr, unless the printer/logging flushes in between. + assert stdout == b"" + print(stderr.decode("utf-8"), end="") + if not conf.no_fmt and not run_command(["cargo", "fmt"]): return 1 print("<= Done!") From af94b64fe7f4fbdebcb622b36cefe3e676a48f4f Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 11 Apr 2021 20:46:42 +0200 Subject: [PATCH 26/32] generator: Simplify error handling Similar to Rusts return and try, just use exceptions and pretty-print them at the highest level (instead of presenting the user with a stacktrace that's harder to read than the error alone). --- generator.py | 62 ++++++++++++++++++++-------------------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/generator.py b/generator.py index a25710991..4ecb15f21 100755 --- a/generator.py +++ b/generator.py @@ -5,24 +5,13 @@ import subprocess import sys - -NOTHING_TO_BE_DONE = 0 -NEED_UPDATE = 1 -FAILURE = 2 - DEFAULT_GIR_FILES_DIRECTORY = Path("./gir-files") DEFAULT_GIR_DIRECTORY = Path("./gir/") DEFAULT_GIR_PATH = DEFAULT_GIR_DIRECTORY / "target/release/gir" def run_command(command, folder=None): - if folder is None: - folder = "." - ret = subprocess.run(command, cwd=folder) - if ret.returncode != 0: - print("Command `{}` failed with `{}`...".format(command, ret)) - return False - return True + return subprocess.run(command, cwd=folder, check=True) def spawn_process(command): @@ -42,37 +31,28 @@ def ask_yes_no_question(question, conf): return line.strip().lower() == "y" -def def_check_submodule(submodule_path, conf): +def update_submodule(submodule_path, conf): if any(submodule_path.iterdir()): - return NOTHING_TO_BE_DONE + return False print("=> Initializing {} submodule...".format(submodule_path)) - if not run_command(["git", "submodule", "update", "--init", submodule_path]): - return FAILURE + run_command(["git", "submodule", "update", "--init", submodule_path]) print("<= Done!") if ask_yes_no_question( "Do you want to update {} submodule?".format(submodule_path), conf ): print("=> Updating submodule...") - if not run_command(["git", "reset", "--hard", "HEAD"], submodule_path): - return FAILURE - if not run_command(["git", "pull", "-f", "origin", "master"], submodule_path): - return FAILURE + run_command(["git", "reset", "--hard", "HEAD"], submodule_path) + run_command(["git", "pull", "-f", "origin", "master"], submodule_path) print("<= Done!") - return NEED_UPDATE - return NOTHING_TO_BE_DONE + return True + return False -def build_gir_if_needed(updated_submodule): - if updated_submodule == FAILURE: - return False +def build_gir(): print("=> Building gir...") - if update_workspace(): - print("<= Done!") - else: - print("<= Failed...") - return False - return True + update_workspace() + print("<= Done!") def regen_crates(path, conf): @@ -190,12 +170,11 @@ def main(): conf = parse_args() if not conf.gir_files_paths: - if def_check_submodule(DEFAULT_GIR_FILES_DIRECTORY, conf) == FAILURE: - return 1 + update_submodule(DEFAULT_GIR_FILES_DIRECTORY, conf) if conf.gir_path == DEFAULT_GIR_PATH: - if not build_gir_if_needed(def_check_submodule(DEFAULT_GIR_DIRECTORY, conf)): - return 1 + update_submodule(DEFAULT_GIR_DIRECTORY, conf) + build_gir() print("=> Regenerating crates...") for path in conf.path: @@ -204,10 +183,13 @@ def main(): for log, p in processes: print("==> {}".format(log)) stdout, stderr = p.communicate() + stdout = stdout.decode("utf-8") + stderr = stderr.decode("utf-8") + assert p.returncode == 0, stderr.strip() # Gir doesn't print anything to stdout. If it does, this is likely out of # order with stderr, unless the printer/logging flushes in between. - assert stdout == b"" - print(stderr.decode("utf-8"), end="") + assert not stdout, "`gir` printed unexpected stdout: {}".format(stdout) + print(stderr, end="") if not conf.no_fmt and not run_command(["cargo", "fmt"]): return 1 @@ -217,4 +199,8 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + try: + main() + except Exception as e: + print("Error: {}".format(e), file=sys.stderr) + sys.exit(1) From 062f7d354c9799308f16282bf47662c7cd14ea94 Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Sun, 18 Apr 2021 17:27:15 +0200 Subject: [PATCH 27/32] generator: Don't generate Rust files when generating docs We want to use the generator in the CI to update docs, for which we don't want to emit updated Rust files at the same time. --- generator.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/generator.py b/generator.py index 4ecb15f21..84156fa22 100755 --- a/generator.py +++ b/generator.py @@ -65,18 +65,18 @@ def regen_crates(path, conf): d for path in conf.gir_files_paths for d in ("-d", path) ] - if path.parent.name.endswith("sys"): - args.extend(["-m", "sys"]) - elif conf.with_docs: + is_sys_crate = path.parent.name.endswith("sys") + + if conf.docs: # Update docs/**/docs.md for non-sys crates + if is_sys_crate: + return processes doc_path = "docs.md" - if isinstance(conf.with_docs, Path): + if isinstance(conf.docs, Path): # doc-target-path is relative to `-c` path_depth = len(path.parent.parts) - doc_path = Path( - *[".."] * path_depth, conf.with_docs, path.parent, doc_path - ) + doc_path = Path(*[".."] * path_depth, conf.docs, path.parent, doc_path) doc_args = args + ["-m", "doc", "--doc-target-path", doc_path] processes.append( ( @@ -86,8 +86,10 @@ def regen_crates(path, conf): spawn_process(doc_args), ) ) - - processes.append(("Regenerating `{}`...".format(path), spawn_process(args))) + else: + if is_sys_crate: + args.extend(["-m", "sys"]) + processes.append(("Regenerating `{}`...".format(path), spawn_process(args))) else: raise Exception("`{}` is not a valid Gir*.toml file".format(path)) @@ -109,6 +111,17 @@ def directory_path(path): return path +def directory_output_path(path): + """ + Creates an output directory if it doesn't exist yet. + + Fails if the directory cannot be created or the path exists but is not a directory. + """ + path = Path(path) + path.mkdir(parents=True, exist_ok=True) + return path + + def file_path(path): path = Path(path) if not path.is_file(): @@ -154,12 +167,12 @@ def parse_args(): help="If set, this script will not run `cargo fmt`", ) parser.add_argument( - "--with-docs", + "--docs", metavar="output_path", nargs="?", const=True, default=False, - type=directory_path, + type=directory_output_path, help="Build documentation with `gir -m doc`. Optionally takes an output directory", ) From 2ec4ab342e34da439f9d581bc3fc9ff384d4bba9 Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Tue, 20 Apr 2021 00:31:23 +0200 Subject: [PATCH 28/32] generator: Convert delayed subprocess communicate() to asyncio Asyncio is much better suited for spawning complex tasks - in this case involving process spawning intermixed with python code - without wrapping everything in custom tuples and arrays. --- generator.py | 75 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/generator.py b/generator.py index 84156fa22..f5d0a67ee 100755 --- a/generator.py +++ b/generator.py @@ -4,6 +4,7 @@ import argparse import subprocess import sys +import asyncio DEFAULT_GIR_FILES_DIRECTORY = Path("./gir-files") DEFAULT_GIR_DIRECTORY = Path("./gir/") @@ -14,8 +15,29 @@ def run_command(command, folder=None): return subprocess.run(command, cwd=folder, check=True) -def spawn_process(command): - return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +async def spawn_process(exe, args): + p = await asyncio.create_subprocess_exec( + exe, + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + stdout, stderr = await p.communicate() + stdout = stdout.decode("utf-8") + stderr = stderr.decode("utf-8") + assert p.returncode == 0, stderr.strip() + return stdout, stderr + + +async def spawn_gir(gir_exe, args): + stdout, stderr = await spawn_process(gir_exe, args) + # Gir doesn't print anything to stdout. If it does, this is likely out of + # order with stderr, unless the printer/logging flushes in between. + assert not stdout, "`gir` printed unexpected stdout: {}".format(stdout) + if stderr: + return "===> stderr:\n\n" + stderr + "\n" + return "" def update_workspace(): @@ -55,13 +77,24 @@ def build_gir(): print("<= Done!") +async def regenerate_crate_docs(gir_exe, crate_dir, base_gir_args, doc_path): + doc_args = base_gir_args + ["-m", "doc", "--doc-target-path", doc_path] + + logs = "==> Regenerating documentation for `{}` into `{}`...\n".format( + crate_dir, doc_path + ) + logs += await spawn_gir(gir_exe, doc_args) + + return logs + + def regen_crates(path, conf): processes = [] if path.is_dir(): for entry in path.rglob("Gir*.toml"): processes += regen_crates(entry, conf) elif path.match("Gir*.toml"): - args = [conf.gir_path, "-c", path, "-o", path.parent] + [ + args = ["-c", path, "-o", path.parent] + [ d for path in conf.gir_files_paths for d in ("-d", path) ] @@ -72,24 +105,26 @@ def regen_crates(path, conf): if is_sys_crate: return processes + # Generate into docs.md instead of the default vendor.md doc_path = "docs.md" if isinstance(conf.docs, Path): # doc-target-path is relative to `-c` path_depth = len(path.parent.parts) doc_path = Path(*[".."] * path_depth, conf.docs, path.parent, doc_path) - doc_args = args + ["-m", "doc", "--doc-target-path", doc_path] + processes.append( - ( - "Regenerating documentation for `{}` into `{}`...".format( - path, doc_path - ), - spawn_process(doc_args), - ) + regenerate_crate_docs(conf.gir_path, path.parent, args, doc_path) ) else: if is_sys_crate: args.extend(["-m", "sys"]) - processes.append(("Regenerating `{}`...".format(path), spawn_process(args))) + + async def regenerate_crate(path, args): + return "==> Regenerating `{}`...\n".format(path) + await spawn_gir( + conf.gir_path, args + ) + + processes.append(regenerate_crate(path, args)) else: raise Exception("`{}` is not a valid Gir*.toml file".format(path)) @@ -179,7 +214,7 @@ def parse_args(): return parser.parse_args() -def main(): +async def main(): conf = parse_args() if not conf.gir_files_paths: @@ -192,17 +227,9 @@ def main(): print("=> Regenerating crates...") for path in conf.path: print("=> Looking in path `{}`".format(path)) - processes = regen_crates(path, conf) - for log, p in processes: - print("==> {}".format(log)) - stdout, stderr = p.communicate() - stdout = stdout.decode("utf-8") - stderr = stderr.decode("utf-8") - assert p.returncode == 0, stderr.strip() - # Gir doesn't print anything to stdout. If it does, this is likely out of - # order with stderr, unless the printer/logging flushes in between. - assert not stdout, "`gir` printed unexpected stdout: {}".format(stdout) - print(stderr, end="") + # Collect and print the results as soon as they trickle in, one process at a time: + for coro in asyncio.as_completed(regen_crates(path, conf)): + print(await coro, end="") if not conf.no_fmt and not run_command(["cargo", "fmt"]): return 1 @@ -213,7 +240,7 @@ def main(): if __name__ == "__main__": try: - main() + asyncio.run(main()) except Exception as e: print("Error: {}".format(e), file=sys.stderr) sys.exit(1) From 2103cbf22de9b5e72e83e800f1d65b1701d927ab Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Tue, 20 Apr 2021 00:33:32 +0200 Subject: [PATCH 29/32] generator: Embed documentation using rustdoc-stripper after generating We have decided to get rid of checked-in documentation in favour of performing generation and embedding at once in the CI (or locally for testing) in a single script. Since the generator is already looping over crates containing Gir.toml this is the only natural place for that. This change removes the custom documentation output path again (leaving it to the default `/docs.md`) and invokes `rustdoc-stripper -g` straight after to embed the documentation. The command-line flag is now a simple boolean to enable this mode; without it Rust files are generated as normal. --- generator.py | 53 ++++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/generator.py b/generator.py index f5d0a67ee..4a8cd400e 100755 --- a/generator.py +++ b/generator.py @@ -77,14 +77,29 @@ def build_gir(): print("<= Done!") -async def regenerate_crate_docs(gir_exe, crate_dir, base_gir_args, doc_path): +async def regenerate_crate_docs(gir_exe, crate_dir, base_gir_args): + doc_path = "docs.md" + # Generate into docs.md instead of the default vendor.md doc_args = base_gir_args + ["-m", "doc", "--doc-target-path", doc_path] + # The above `gir -m doc` generates docs.md relative to the directory containing Gir.toml + doc_path = crate_dir / doc_path + embed_args = ["-g", "-m", "-o", doc_path, "-d", crate_dir / "src"] + logs = "==> Regenerating documentation for `{}` into `{}`...\n".format( crate_dir, doc_path ) logs += await spawn_gir(gir_exe, doc_args) + logs += "==> Embedding documentation from `{}` into `{}`...\n".format( + doc_path, crate_dir + ) + stdout, stderr = await spawn_process("rustdoc-stripper", embed_args) + if stdout: + logs += "===> stdout:\n\n" + stdout + "\n" + if stderr: + logs += "===> stderr:\n\n" + stderr + "\n" + return logs @@ -100,21 +115,12 @@ def regen_crates(path, conf): is_sys_crate = path.parent.name.endswith("sys") - if conf.docs: - # Update docs/**/docs.md for non-sys crates + if conf.embed_docs: + # Embedding documentation only applies to non-sys crates if is_sys_crate: return processes - # Generate into docs.md instead of the default vendor.md - doc_path = "docs.md" - if isinstance(conf.docs, Path): - # doc-target-path is relative to `-c` - path_depth = len(path.parent.parts) - doc_path = Path(*[".."] * path_depth, conf.docs, path.parent, doc_path) - - processes.append( - regenerate_crate_docs(conf.gir_path, path.parent, args, doc_path) - ) + processes.append(regenerate_crate_docs(conf.gir_path, path.parent, args)) else: if is_sys_crate: args.extend(["-m", "sys"]) @@ -146,17 +152,6 @@ def directory_path(path): return path -def directory_output_path(path): - """ - Creates an output directory if it doesn't exist yet. - - Fails if the directory cannot be created or the path exists but is not a directory. - """ - path = Path(path) - path.mkdir(parents=True, exist_ok=True) - return path - - def file_path(path): path = Path(path) if not path.is_file(): @@ -202,13 +197,9 @@ def parse_args(): help="If set, this script will not run `cargo fmt`", ) parser.add_argument( - "--docs", - metavar="output_path", - nargs="?", - const=True, - default=False, - type=directory_output_path, - help="Build documentation with `gir -m doc`. Optionally takes an output directory", + "--embed-docs", + action="store_true", + help="Build documentation with `gir -m doc`, and embed it with `rustdoc-stripper -g`", ) return parser.parse_args() From 97c80d07466928cc4c41406efbf36f6e198002f7 Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Wed, 21 Apr 2021 00:24:36 +0200 Subject: [PATCH 30/32] generator: Convert PosixPath to str for Python 3.7 asyncio This is fixed in Python 3.8: https://github.com/python/cpython/commit/744c08a9c75a1a53b7a6521fcee3e7c513919ff9 --- generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generator.py b/generator.py index 4a8cd400e..8931d99d3 100755 --- a/generator.py +++ b/generator.py @@ -17,8 +17,8 @@ def run_command(command, folder=None): async def spawn_process(exe, args): p = await asyncio.create_subprocess_exec( - exe, - *args, + str(exe), + *(str(arg) for arg in args), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) From 006ec07351ef7bb624172d085f2c18521a69c555 Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Wed, 14 Apr 2021 10:59:54 +0200 Subject: [PATCH 31/32] cargo: Exclude generator.py from the crate --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f7f332189..1f0e9b4ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ authors = [ "Guillaume Gomez ", ] build = "build.rs" -exclude = ["Gir*.toml", "tests/**/*", "*.md"] +exclude = ["Gir*.toml", "tests/**/*", "*.md", "generator.py"] edition = "2018" [dependencies] From c423525068d4c46c7e1afa088d87f76e7bd45cc0 Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Wed, 21 Apr 2021 00:28:36 +0200 Subject: [PATCH 32/32] README: Document usage of `./generator.py` --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 578c7bbfb..63cb2d34e 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ The `GIR` is used to generate both the sys level crate and a safe API crate to u This README files is more about the options and little overview. If you want a tutorial on how to generate a crate using `gir`, we recommend you to read this [tutorial](https://gtk-rs.org/docs-src/tutorial/gir_tutorial) instead. +`gir` includes a wrapper script `./generator.py` that detects `Gir.toml` configurations in the current directory (or the path(s) passed on the command-line) and generates "normal" or "sys" crates for it. Alternatively `--embed-docs` can be passed to prepare source-code for a documentation build by moving all documentation into it. For a complete overview of available options, pass `--help`. + ## Introduction to `gir` generation Using `gir` requires both a `*.toml` and a `*.gir` for generation of the bindings. @@ -153,7 +155,7 @@ trust_return_value_nullability = false # Disable running `cargo fmt` on generated files # (defaults to false) disable_format = true -# Always generate a Builder if possible. This is mostly a convenient setter as most of the +# Always generate a Builder if possible. This is mostly a convenient setter as most of the # time you might want the Builder to be generated. Ignoring none-desired ones can still be done with per object `generate_builder` configuration. # (defaults to false) generate_builder = true