New pypi wait strategy to fix release pipeline.

codebuild/cd/publish_to_prod_pypi.yml

@@ -22,7 +22,6 @@ phases:
       - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
       - python3 continuous-delivery/pull-pypirc.py prod
       - python3 -m twine upload -r pypi ../dist/*
-      - python3 continuous-delivery/wait-for-pypi.py awscrt $CURRENT_TAG_VERSION --index https://pypi.python.org/pypi
   post_build:
     commands:
       - echo Build completed on `date`

codebuild/cd/publish_to_test_pypi.yml

@@ -22,7 +22,6 @@ phases:
       - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
       - python3 continuous-delivery/pull-pypirc.py alpha
       - python3 -m twine upload -r testpypi ../dist/*
-      - python3 continuous-delivery/wait-for-pypi.py awscrt $CURRENT_TAG_VERSION --index https://test.pypi.org/pypi
   post_build:
     commands:
       - echo Build completed on `date`

codebuild/cd/test_prod_pypi.yml

@@ -14,7 +14,7 @@ phases:
       - echo Build started on `date`
       - cd aws-crt-python
       - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
-      - python3 -m pip install --no-cache-dir --user awscrt==$CURRENT_TAG_VERSION
+      - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user awscrt==$CURRENT_TAG_VERSION
       - python3 continuous-delivery/test-pip-install.py
   post_build:
     commands:

codebuild/cd/test_test_pypi.yml

@@ -14,7 +14,7 @@ phases:
       - echo Build started on `date`
       - cd aws-crt-python
       - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
-      - python3 -m pip install --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION
+      - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION
       - python3 continuous-delivery/test-pip-install.py
   post_build:
     commands:

continuous-delivery/pip-install-with-retry.py

@@ -0,0 +1,39 @@
+import time
+import sys
+import subprocess
+
+DOCS = """Given cmdline args, executes: python3 -m pip install [args...]
+Keeps retrying until the new version becomes available in pypi (or we time out)"""
+if len(sys.argv) < 2:
+    sys.exit(DOCS)
+
+RETRY_INTERVAL_SECS = 10
+GIVE_UP_AFTER_SECS = 60 * 15
+
+pip_install_args = [sys.executable, '-m', 'pip', 'install'] + sys.argv[1:]
+
+start_time = time.time()
+while True:
+    print(subprocess.list2cmdline(pip_install_args))
+    result = subprocess.run(pip_install_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+    stdout = result.stdout.decode().strip()
+    if stdout:
+        print(stdout)
+
+    if result.returncode == 0:
+        # success
+        sys.exit(0)
+
+    if "could not find a version" in stdout.lower():
+        elapsed_secs = time.time() - start_time
+        if elapsed_secs < GIVE_UP_AFTER_SECS:
+            # try again
+            print("Retrying in", RETRY_INTERVAL_SECS, "secs...")
+            time.sleep(RETRY_INTERVAL_SECS)
+            continue
+        else:
+            print("Giving up on retries after", int(elapsed_secs), "total secs.")
+
+    # fail
+    sys.exit(result.returncode)

continuous-delivery/sanity-check-test-pypi.bat

@@ -1,7 +1,7 @@
 FOR /F "delims=" %%A in ('git describe --tags') do ( set TAG_VERSION=%%A )
 set CURRENT_VERSION=%TAG_VERSION:v=%
 
-"C:\Program Files\Python37\python.exe" -m pip install --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==%CURRENT_VERSION% || goto error
+"C:\Program Files\Python37\python.exe" continuous-delivery\pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==%CURRENT_VERSION% || goto error
 "C:\Program Files\Python37\python.exe" continuous-delivery\test-pip-install.py || goto error
 
 goto :EOF

continuous-delivery/sanity-check-test-pypi.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 set -ex
 CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
-python3 -m pip install --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION
+python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir -i https://testpypi.python.org/simple --user awscrt==$CURRENT_TAG_VERSION
 python3 continuous-delivery/test-pip-install.py