From f9d5ab2f8936df11b7ccc420d3563f7f0672d180 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Thu, 7 Jan 2021 22:10:24 -0800 Subject: [PATCH] New pypi wait strategy to fix release pipeline. Our release pipeline needs to wait until a newly uploaded version is available in pypi, before installing the new version for testing. We were using `pip search` to determine that the new version was available, but `pip search` has been disabled at a server level (https://status.python.org/incidents/grk0k7sz6zkp). We've had issues with this approach in the past anyway, where it looked like it was globally available, but then didn't install on some specific platform. New strategy is to just retry install attempts until they work (or cap out). --- codebuild/cd/publish_to_prod_pypi.yml | 1 - codebuild/cd/publish_to_test_pypi.yml | 1 - codebuild/cd/test_prod_pypi.yml | 2 +- codebuild/cd/test_test_pypi.yml | 2 +- continuous-delivery/pip-install-with-retry.py | 39 +++++++++++++++++ .../sanity-check-test-pypi.bat | 2 +- continuous-delivery/sanity-check-test-pypi.sh | 2 +- continuous-delivery/wait-for-pypi.py | 43 ------------------- 8 files changed, 43 insertions(+), 49 deletions(-) create mode 100644 continuous-delivery/pip-install-with-retry.py delete mode 100644 continuous-delivery/wait-for-pypi.py diff --git a/codebuild/cd/publish_to_prod_pypi.yml b/codebuild/cd/publish_to_prod_pypi.yml index ef3e9c2c9..8d44ad32d 100644 --- a/codebuild/cd/publish_to_prod_pypi.yml +++ b/codebuild/cd/publish_to_prod_pypi.yml @@ -22,7 +22,6 @@ phases: - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) - python3 continuous-delivery/pull-pypirc.py prod - python3 -m twine upload -r pypi ../dist/* - - python3 continuous-delivery/wait-for-pypi.py awscrt $CURRENT_TAG_VERSION --index https://pypi.python.org/pypi post_build: commands: - echo Build completed on `date` diff --git a/codebuild/cd/publish_to_test_pypi.yml b/codebuild/cd/publish_to_test_pypi.yml index 1c4416ec4..c245b5862 100644 --- a/codebuild/cd/publish_to_test_pypi.yml +++ b/codebuild/cd/publish_to_test_pypi.yml @@ -22,7 +22,6 @@ phases: - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) - python3 continuous-delivery/pull-pypirc.py alpha - python3 -m twine upload -r testpypi ../dist/* - - python3 continuous-delivery/wait-for-pypi.py awscrt $CURRENT_TAG_VERSION --index https://test.pypi.org/pypi post_build: commands: - echo Build completed on `date` diff --git a/codebuild/cd/test_prod_pypi.yml b/codebuild/cd/test_prod_pypi.yml index 0ed99ee3c..f8ee615a3 100644 --- a/codebuild/cd/test_prod_pypi.yml +++ b/codebuild/cd/test_prod_pypi.yml @@ -14,7 +14,7 @@ phases: - echo Build started on `date` - cd aws-crt-python - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) - - python3 -m pip install --no-cache-dir --user awscrt==$CURRENT_TAG_VERSION + - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user awscrt==$CURRENT_TAG_VERSION - python3 continuous-delivery/test-pip-install.py post_build: commands: diff --git a/codebuild/cd/test_test_pypi.yml b/codebuild/cd/test_test_pypi.yml index 1e9a46169..7362873aa 100644 --- a/codebuild/cd/test_test_pypi.yml +++ b/codebuild/cd/test_test_pypi.yml @@ -14,7 +14,7 @@ phases: - echo Build started on `date` - cd aws-crt-python - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) - - python3 -m pip install --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION + - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION - python3 continuous-delivery/test-pip-install.py post_build: commands: diff --git a/continuous-delivery/pip-install-with-retry.py b/continuous-delivery/pip-install-with-retry.py new file mode 100644 index 000000000..347e0dca9 --- /dev/null +++ b/continuous-delivery/pip-install-with-retry.py @@ -0,0 +1,39 @@ +import time +import sys +import subprocess + +DOCS = """Given cmdline args, executes: python3 -m pip install [args...] +Keeps retrying until the new version becomes available in pypi (or we time out)""" +if len(sys.argv) < 2: + sys.exit(DOCS) + +RETRY_INTERVAL_SECS = 10 +GIVE_UP_AFTER_SECS = 60 * 15 + +pip_install_args = [sys.executable, '-m', 'pip', 'install'] + sys.argv[1:] + +start_time = time.time() +while True: + print(subprocess.list2cmdline(pip_install_args)) + result = subprocess.run(pip_install_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + stdout = result.stdout.decode().strip() + if stdout: + print(stdout) + + if result.returncode == 0: + # success + sys.exit(0) + + if "could not find a version" in stdout.lower(): + elapsed_secs = time.time() - start_time + if elapsed_secs < GIVE_UP_AFTER_SECS: + # try again + print("Retrying in", RETRY_INTERVAL_SECS, "secs...") + time.sleep(RETRY_INTERVAL_SECS) + continue + else: + print("Giving up on retries after", int(elapsed_secs), "total secs.") + + # fail + sys.exit(result.returncode) diff --git a/continuous-delivery/sanity-check-test-pypi.bat b/continuous-delivery/sanity-check-test-pypi.bat index 34bd2e67b..949911612 100644 --- a/continuous-delivery/sanity-check-test-pypi.bat +++ b/continuous-delivery/sanity-check-test-pypi.bat @@ -1,7 +1,7 @@ FOR /F "delims=" %%A in ('git describe --tags') do ( set TAG_VERSION=%%A ) set CURRENT_VERSION=%TAG_VERSION:v=% -"C:\Program Files\Python37\python.exe" -m pip install --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==%CURRENT_VERSION% || goto error +"C:\Program Files\Python37\python.exe" continuous-delivery\pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==%CURRENT_VERSION% || goto error "C:\Program Files\Python37\python.exe" continuous-delivery\test-pip-install.py || goto error goto :EOF diff --git a/continuous-delivery/sanity-check-test-pypi.sh b/continuous-delivery/sanity-check-test-pypi.sh index 97c86ec82..6c0f86fe6 100644 --- a/continuous-delivery/sanity-check-test-pypi.sh +++ b/continuous-delivery/sanity-check-test-pypi.sh @@ -1,5 +1,5 @@ #!/bin/bash set -ex CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv) -python3 -m pip install --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION +python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION python3 continuous-delivery/test-pip-install.py diff --git a/continuous-delivery/wait-for-pypi.py b/continuous-delivery/wait-for-pypi.py deleted file mode 100644 index ff2124b2d..000000000 --- a/continuous-delivery/wait-for-pypi.py +++ /dev/null @@ -1,43 +0,0 @@ -import argparse -import subprocess -import sys -import time - -DEFAULT_TIMEOUT = 60 * 30 # 30 min -DEFAULT_INTERVAL = 5 -DEFAULT_INDEX_URL = 'https://pypi.python.org/pypi' - - -def wait(package, version, index_url=DEFAULT_INDEX_URL, timeout=DEFAULT_TIMEOUT, interval=DEFAULT_INTERVAL): - give_up_time = time.time() + timeout - while True: - output = subprocess.check_output([sys.executable, '-m', 'pip', 'search', - '--no-cache-dir', '--index', index_url, package]) - output = output.decode() - - # output looks like: 'awscrt (0.3.1) - A common runtime for AWS Python projects\n...' - if output.startswith('{} ({})'.format(package, version)): - return True - - if time.time() >= give_up_time: - return False - - time.sleep(interval) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('package', help="Packet name") - parser.add_argument('version', help="Package version") - parser.add_argument('--index', default=DEFAULT_INDEX_URL, metavar='', - help="Base URL of Python Package Index. (default {})".format(DEFAULT_INDEX_URL)) - parser.add_argument('--timeout', type=float, default=DEFAULT_TIMEOUT, metavar='', - help="Give up after N seconds.") - parser.add_argument('--interval', type=float, default=DEFAULT_INTERVAL, metavar='', - help="Query PyPI every N seconds") - args = parser.parse_args() - - if wait(args.package, args.version, args.index, args.timeout, args.interval): - print('{} {} is available in pypi'.format(args.package, args.version)) - else: - exit("Timed out waiting for pypi to report {} {} as latest".format(args.package, args.version))