Metadata: Retry dockerhub fetch on metadata post publish fail (#32626)

Co-authored-by: bnchrch <bnchrch@users.noreply.github.com>

airbyte-ci/connectors/metadata_service/lib/metadata_service/docker_hub.py

@@ -3,6 +3,7 @@
 #
 
 import os
+import time
 from typing import Optional
 
 import requests
@@ -26,23 +27,34 @@ def get_docker_hub_auth_token() -> str:
     return token
 
 
-def is_image_on_docker_hub(image_name: str, version: str, digest: Optional[str] = None) -> bool:
+def is_image_on_docker_hub(image_name: str, version: str, digest: Optional[str] = None, retries: int = 0, wait_sec: int = 30) -> bool:
     """Check if a given image and version exists on Docker Hub.
 
     Args:
         image_name (str): The name of the image to check.
         version (str): The version of the image to check.
         digest (str, optional): The digest of the image to check. Defaults to None.
+        retries (int, optional): The number of times to retry the request. Defaults to 0.
+        wait_sec (int, optional): The number of seconds to wait between retries. Defaults to 30.
     Returns:
         bool: True if the image and version exists on Docker Hub, False otherwise.
     """
 
     token = get_docker_hub_auth_token()
     headers = {"Authorization": f"JWT {token}"}
     tag_url = f"https://registry.hub.docker.com/v2/repositories/{image_name}/tags/{version}"
-    response = requests.get(tag_url, headers=headers)
+
+    # Allow for retries as the DockerHub API is not always reliable with returning the latest publish.
+    for _ in range(retries + 1):
+        response = requests.get(tag_url, headers=headers)
+        if response.ok:
+            break
+        time.sleep(wait_sec)
+
     if not response.ok:
+        response.raise_for_status()
         return False
+
     # If a digest is provided, check that it matches the digest of the image on Docker Hub.
     if digest is not None:
         return f"sha256:{digest}" == response.json()["digest"]

airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py

@@ -164,7 +164,8 @@ def validate_metadata_base_images_in_dockerhub(
     tag = tag_with_sha_prefix.split("@")[0]
 
     print(f"Checking that the base images is on dockerhub: {image_address}")
-    if not is_image_on_docker_hub(image_name, tag, digest):
+
+    if not is_image_on_docker_hub(image_name, tag, digest, retries=3):
         return False, f"Image {image_address} does not exist in DockerHub"
 
     return True, None