From 42499f1b5340bf8d6dc89b102ad3c9f94ea59396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sat, 18 Nov 2023 22:31:59 +0200 Subject: [PATCH] feat: generator option to skip digest verify where applicable Use argparse for consistency in others, too. --- generators/README.md | 4 ++++ generators/wrun_black_args.py | 10 +++++----- generators/wrun_committed_args.py | 27 +++++++++++++++++---------- generators/wrun_dprint_args.py | 19 +++++++++++-------- generators/wrun_golangci_lint_args.py | 19 +++++++++++-------- generators/wrun_ruff_args.py | 19 +++++++++++-------- generators/wrun_shellcheck_args.py | 10 +++++----- generators/wrun_shfmt_args.py | 10 +++++----- generators/wrun_terraform_args.py | 19 +++++++++++-------- generators/wrun_tflint_args.py | 19 +++++++++++-------- generators/wrun_typos_args.py | 27 +++++++++++++++++---------- generators/wrun_vacuum_args.py | 19 +++++++++++-------- 12 files changed, 119 insertions(+), 83 deletions(-) diff --git a/generators/README.md b/generators/README.md index b35f10e..f9c33a9 100644 --- a/generators/README.md +++ b/generators/README.md @@ -28,6 +28,10 @@ Hint: if embedding to a YAML document as a string, e.g. a CI config, using [line folding (`>-`)](https://yaml.org/spec/1.2.2/#65-line-folding) the readability can likely be preserved there, too. +Some of the scripts operate on upstream provided checksum files. +They have an option to skip verifying checksums against the actual payloads at +the executable or archive URLs, `--skip-verify`. + ## TODO Some tools for which generators would be nice to have, contributions welcome! diff --git a/generators/wrun_black_args.py b/generators/wrun_black_args.py index 12b784b..85cbedf 100755 --- a/generators/wrun_black_args.py +++ b/generators/wrun_black_args.py @@ -23,8 +23,8 @@ * https://github.com/psf/black/releases """ +from argparse import ArgumentParser import hashlib -import sys from urllib.parse import urljoin, quote as urlquote from urllib.request import urlopen @@ -47,7 +47,7 @@ def main(version: str) -> None: if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + args = parser.parse_args() + main(args.version) diff --git a/generators/wrun_committed_args.py b/generators/wrun_committed_args.py index 301a124..29870e4 100755 --- a/generators/wrun_committed_args.py +++ b/generators/wrun_committed_args.py @@ -24,9 +24,9 @@ * https://warehouse.pypa.io/api-reference/json.html """ +from argparse import ArgumentParser import hashlib import json -import sys from urllib.parse import quote as urlquote from urllib.request import urlopen @@ -43,24 +43,28 @@ } -def check_hexdigest(url: str, algo: str, expected: str) -> None: +def check_hexdigest(expected: str, algo: str, url: str | None) -> None: try: assert len(expected) == len(hashlib.new(algo, b"canary").hexdigest()) _ = bytes.fromhex(expected) except Exception as e: raise ValueError(f'not a {algo} hex digest: "{expected}"') from e + if not url: + return with urlopen(url) as f: got = hashlib.file_digest(f, algo).hexdigest() if got != expected: raise ValueError(f'{algo} mismatch for "{url}", expected {expected}, got {got}') -def main(version: str) -> None: +def main(version: str, verify: bool) -> None: project = "committed" version_number = version.lstrip("v") archive_exe_paths = [] - release_url = f"https://pypi.org/pypi/{urlquote(project)}/{urlquote(version_number)}/json" + release_url = ( + f"https://pypi.org/pypi/{urlquote(project)}/{urlquote(version_number)}/json" + ) with urlopen(release_url) as f: release_data = json.load(f) @@ -75,14 +79,16 @@ def main(version: str) -> None: except KeyError as e: raise KeyError(f"no sha256 digest available for {filename}") from e - lookup_filename = filename.replace(f"-{version_number}-", "-{version_number}-", 1) + lookup_filename = filename.replace( + f"-{version_number}-", "-{version_number}-", 1 + ) if lookup_filename not in file_os_archs: raise KeyError(f'unhandled file: "{filename}"') os_arch = file_os_archs[lookup_filename] if os_arch is None: continue - check_hexdigest(url["url"], "sha256", hexdigest) + check_hexdigest(hexdigest, "sha256", url["url"] if verify else None) print(f"-url {os_arch}={url['url']}#sha256-{hexdigest}") suffix = ".exe" if os_arch.startswith("windows/") else "" @@ -94,7 +100,8 @@ def main(version: str) -> None: if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + parser.add_argument("--skip-verify", dest="verify", action="store_false") + args = parser.parse_args() + main(args.version, args.verify) diff --git a/generators/wrun_dprint_args.py b/generators/wrun_dprint_args.py index 281a890..3b66942 100755 --- a/generators/wrun_dprint_args.py +++ b/generators/wrun_dprint_args.py @@ -23,8 +23,8 @@ * https://github.com/dprint/dprint/releases """ +from argparse import ArgumentParser import hashlib -import sys from urllib.parse import urljoin, quote as urlquote from urllib.request import urlopen @@ -40,19 +40,21 @@ } -def check_hexdigest(url: str, algo: str, expected: str) -> None: +def check_hexdigest(expected: str, algo: str, url: str | None) -> None: try: assert len(expected) == len(hashlib.new(algo, b"canary").hexdigest()) _ = bytes.fromhex(expected) except Exception as e: raise ValueError(f'not a {algo} hex digest: "{expected}"') from e + if not url: + return with urlopen(url) as f: got = hashlib.file_digest(f, algo).hexdigest() if got != expected: raise ValueError(f'{algo} mismatch for "{url}", expected {expected}, got {got}') -def main(version: str) -> None: +def main(version: str, verify: bool) -> None: base_url = ( f"https://github.com/dprint/dprint/releases/download/{urlquote(version)}/" ) @@ -73,14 +75,15 @@ def main(version: str) -> None: continue url = urljoin(base_url, filename) - check_hexdigest(url, "sha256", hexdigest) + check_hexdigest(hexdigest, "sha256", url if verify else None) print(f"-url {os_arch}={url}#sha256-{hexdigest}") print("-archive-exe-path dprint") if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + parser.add_argument("--skip-verify", dest="verify", action="store_false") + args = parser.parse_args() + main(args.version, args.verify) diff --git a/generators/wrun_golangci_lint_args.py b/generators/wrun_golangci_lint_args.py index 448a614..eb1eec5 100755 --- a/generators/wrun_golangci_lint_args.py +++ b/generators/wrun_golangci_lint_args.py @@ -23,9 +23,9 @@ * https://github.com/golangci/golangci-lint/releases """ +from argparse import ArgumentParser import hashlib import re -import sys from urllib.parse import urljoin, quote as urlquote from urllib.request import urlopen @@ -61,19 +61,21 @@ } -def check_hexdigest(url: str, algo: str, expected: str) -> None: +def check_hexdigest(expected: str, algo: str, url: str | None) -> None: try: assert len(expected) == len(hashlib.new(algo, b"canary").hexdigest()) _ = bytes.fromhex(expected) except Exception as e: raise ValueError(f'not a {algo} hex digest: "{expected}"') from e + if not url: + return with urlopen(url) as f: got = hashlib.file_digest(f, algo).hexdigest() if got != expected: raise ValueError(f'{algo} mismatch for "{url}", expected {expected}, got {got}') -def main(version: str) -> None: +def main(version: str, verify: bool) -> None: version_number = version.lstrip("v") base_url = f"https://github.com/golangci/golangci-lint/releases/download/{urlquote(version)}/" archive_exe_paths = [] @@ -102,7 +104,7 @@ def main(version: str) -> None: continue url = urljoin(base_url, filename) - check_hexdigest(url, "sha256", hexdigest) + check_hexdigest(hexdigest, "sha256", url if verify else None) print(f"-url {os_arch}={url}#sha256-{hexdigest}") dirname = re.sub(r"\.(t[\w.]+|zip)$", "", filename) @@ -115,7 +117,8 @@ def main(version: str) -> None: if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + parser.add_argument("--skip-verify", dest="verify", action="store_false") + args = parser.parse_args() + main(args.version, args.verify) diff --git a/generators/wrun_ruff_args.py b/generators/wrun_ruff_args.py index 1608cac..464812d 100755 --- a/generators/wrun_ruff_args.py +++ b/generators/wrun_ruff_args.py @@ -24,8 +24,8 @@ * https://github.com/astral-sh/ruff/releases """ +from argparse import ArgumentParser import hashlib -import sys from urllib.parse import urljoin, quote as urlquote from urllib.request import urlopen @@ -49,19 +49,21 @@ } -def check_hexdigest(url: str, algo: str, expected: str) -> None: +def check_hexdigest(expected: str, algo: str, url: str | None) -> None: try: assert len(expected) == len(hashlib.new(algo, b"canary").hexdigest()) _ = bytes.fromhex(expected) except Exception as e: raise ValueError(f'not a {algo} hex digest: "{expected}"') from e + if not url: + return with urlopen(url) as f: got = hashlib.file_digest(f, algo).hexdigest() if got != expected: raise ValueError(f'{algo} mismatch for "{url}", expected {expected}, got {got}') -def main(version: str) -> None: +def main(version: str, verify: bool) -> None: base_url = ( f"https://github.com/astral-sh/ruff/releases/download/{urlquote(version)}/" ) @@ -86,14 +88,15 @@ def main(version: str) -> None: continue url = urljoin(base_url, filename) - check_hexdigest(url, "sha256", hexdigest) + check_hexdigest(hexdigest, "sha256", url if verify else None) print(f"-url {os_arch}={url}#sha256-{hexdigest}") print("-archive-exe-path ruff") if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + parser.add_argument("--skip-verify", dest="verify", action="store_false") + args = parser.parse_args() + main(args.version, args.verify) diff --git a/generators/wrun_shellcheck_args.py b/generators/wrun_shellcheck_args.py index 7a048bb..5edc411 100755 --- a/generators/wrun_shellcheck_args.py +++ b/generators/wrun_shellcheck_args.py @@ -23,8 +23,8 @@ * https://github.com/koalaman/shellcheck/releases """ +from argparse import ArgumentParser import hashlib -import sys from urllib.parse import urljoin, quote as urlquote from urllib.request import urlopen @@ -53,7 +53,7 @@ def main(version: str) -> None: if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + args = parser.parse_args() + main(args.version) diff --git a/generators/wrun_shfmt_args.py b/generators/wrun_shfmt_args.py index 4689c52..351835a 100755 --- a/generators/wrun_shfmt_args.py +++ b/generators/wrun_shfmt_args.py @@ -23,8 +23,8 @@ * https://github.com/mvdan/sh/releases """ +from argparse import ArgumentParser import hashlib -import sys from urllib.parse import urljoin, quote as urlquote from urllib.request import urlopen @@ -52,7 +52,7 @@ def main(version: str) -> None: if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + args = parser.parse_args() + main(args.version) diff --git a/generators/wrun_terraform_args.py b/generators/wrun_terraform_args.py index 24669bd..e6a3fbe 100755 --- a/generators/wrun_terraform_args.py +++ b/generators/wrun_terraform_args.py @@ -23,8 +23,8 @@ * https://developer.hashicorp.com/terraform/install """ +from argparse import ArgumentParser import hashlib -import sys from urllib.parse import urljoin, quote as urlquote from urllib.request import urlopen @@ -46,19 +46,21 @@ } -def check_hexdigest(url: str, algo: str, expected: str) -> None: +def check_hexdigest(expected: str, algo: str, url: str | None) -> None: try: assert len(expected) == len(hashlib.new(algo, b"canary").hexdigest()) _ = bytes.fromhex(expected) except Exception as e: raise ValueError(f'not a {algo} hex digest: "{expected}"') from e + if not url: + return with urlopen(url) as f: got = hashlib.file_digest(f, algo).hexdigest() if got != expected: raise ValueError(f'{algo} mismatch for "{url}", expected {expected}, got {got}') -def main(version: str) -> None: +def main(version: str, verify: bool) -> None: base_url = f"https://releases.hashicorp.com/terraform/{urlquote(version)}/" with urlopen(urljoin(base_url, urlquote(f"terraform_{version}_SHA256SUMS"))) as f: @@ -78,14 +80,15 @@ def main(version: str) -> None: continue url = urljoin(base_url, filename) - check_hexdigest(url, "sha256", hexdigest) + check_hexdigest(hexdigest, "sha256", url if verify else None) print(f"-url {os_arch}={url}#sha256-{hexdigest}") print("-archive-exe-path terraform") if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + parser.add_argument("--skip-verify", dest="verify", action="store_false") + args = parser.parse_args() + main(args.version, args.verify) diff --git a/generators/wrun_tflint_args.py b/generators/wrun_tflint_args.py index e090d3a..29944b9 100755 --- a/generators/wrun_tflint_args.py +++ b/generators/wrun_tflint_args.py @@ -23,8 +23,8 @@ * https://github.com/terraform-linters/tflint/releases """ +from argparse import ArgumentParser import hashlib -import sys from urllib.parse import urljoin, quote as urlquote from urllib.request import urlopen @@ -40,19 +40,21 @@ } -def check_hexdigest(url: str, algo: str, expected: str) -> None: +def check_hexdigest(expected: str, algo: str, url: str | None) -> None: try: assert len(expected) == len(hashlib.new(algo, b"canary").hexdigest()) _ = bytes.fromhex(expected) except Exception as e: raise ValueError(f'not a {algo} hex digest: "{expected}"') from e + if not url: + return with urlopen(url) as f: got = hashlib.file_digest(f, algo).hexdigest() if got != expected: raise ValueError(f'{algo} mismatch for "{url}", expected {expected}, got {got}') -def main(version: str) -> None: +def main(version: str, verify: bool) -> None: base_url = f"https://github.com/terraform-linters/tflint/releases/download/{urlquote(version)}/" with urlopen(urljoin(base_url, "checksums.txt")) as f: @@ -71,14 +73,15 @@ def main(version: str) -> None: continue url = urljoin(base_url, filename) - check_hexdigest(url, "sha256", hexdigest) + check_hexdigest(hexdigest, "sha256", url if verify else None) print(f"-url {os_arch}={url}#sha256-{hexdigest}") print("-archive-exe-path tflint") if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + parser.add_argument("--skip-verify", dest="verify", action="store_false") + args = parser.parse_args() + main(args.version, args.verify) diff --git a/generators/wrun_typos_args.py b/generators/wrun_typos_args.py index 1d3c5d3..e8ee33e 100755 --- a/generators/wrun_typos_args.py +++ b/generators/wrun_typos_args.py @@ -24,9 +24,9 @@ * https://warehouse.pypa.io/api-reference/json.html """ +from argparse import ArgumentParser import hashlib import json -import sys from urllib.parse import quote as urlquote from urllib.request import urlopen @@ -43,24 +43,28 @@ } -def check_hexdigest(url: str, algo: str, expected: str) -> None: +def check_hexdigest(expected: str, algo: str, url: str | None) -> None: try: assert len(expected) == len(hashlib.new(algo, b"canary").hexdigest()) _ = bytes.fromhex(expected) except Exception as e: raise ValueError(f'not a {algo} hex digest: "{expected}"') from e + if not url: + return with urlopen(url) as f: got = hashlib.file_digest(f, algo).hexdigest() if got != expected: raise ValueError(f'{algo} mismatch for "{url}", expected {expected}, got {got}') -def main(version: str) -> None: +def main(version: str, verify: bool) -> None: project = "typos" version_number = version.lstrip("v") archive_exe_paths = [] - release_url = f"https://pypi.org/pypi/{urlquote(project)}/{urlquote(version_number)}/json" + release_url = ( + f"https://pypi.org/pypi/{urlquote(project)}/{urlquote(version_number)}/json" + ) with urlopen(release_url) as f: release_data = json.load(f) @@ -75,14 +79,16 @@ def main(version: str) -> None: except KeyError as e: raise KeyError(f"no sha256 digest available for {filename}") from e - lookup_filename = filename.replace(f"-{version_number}-", "-{version_number}-", 1) + lookup_filename = filename.replace( + f"-{version_number}-", "-{version_number}-", 1 + ) if lookup_filename not in file_os_archs: raise KeyError(f'unhandled file: "{filename}"') os_arch = file_os_archs[lookup_filename] if os_arch is None: continue - check_hexdigest(url["url"], "sha256", hexdigest) + check_hexdigest(hexdigest, "sha256", url["url"] if verify else None) print(f"-url {os_arch}={url['url']}#sha256-{hexdigest}") suffix = ".exe" if os_arch.startswith("windows/") else "" @@ -94,7 +100,8 @@ def main(version: str) -> None: if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + parser.add_argument("--skip-verify", dest="verify", action="store_false") + args = parser.parse_args() + main(args.version, args.verify) diff --git a/generators/wrun_vacuum_args.py b/generators/wrun_vacuum_args.py index bf298c7..8ecb3e4 100755 --- a/generators/wrun_vacuum_args.py +++ b/generators/wrun_vacuum_args.py @@ -23,8 +23,8 @@ * https://github.com/daveshanley/vacuum/releases """ +from argparse import ArgumentParser import hashlib -import sys from urllib.parse import urljoin, quote as urlquote from urllib.request import urlopen @@ -40,19 +40,21 @@ } -def check_hexdigest(url: str, algo: str, expected: str) -> None: +def check_hexdigest(expected: str, algo: str, url: str | None) -> None: try: assert len(expected) == len(hashlib.new(algo, b"canary").hexdigest()) _ = bytes.fromhex(expected) except Exception as e: raise ValueError(f'not a {algo} hex digest: "{expected}"') from e + if not url: + return with urlopen(url) as f: got = hashlib.file_digest(f, algo).hexdigest() if got != expected: raise ValueError(f'{algo} mismatch for "{url}", expected {expected}, got {got}') -def main(version: str) -> None: +def main(version: str, verify: bool) -> None: version_number = version.lstrip("v") base_url = ( f"https://github.com/daveshanley/vacuum/releases/download/{urlquote(version)}/" @@ -77,14 +79,15 @@ def main(version: str) -> None: continue url = urljoin(base_url, filename) - check_hexdigest(url, "sha256", hexdigest) + check_hexdigest(hexdigest, "sha256", url if verify else None) print(f"-url {os_arch}={url}#sha256-{hexdigest}") print("-archive-exe-path vacuum") if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} VERSION") - sys.exit(2) - main(sys.argv[1]) + parser = ArgumentParser() + parser.add_argument("version", metavar="VERSION") + parser.add_argument("--skip-verify", dest="verify", action="store_false") + args = parser.parse_args() + main(args.version, args.verify)