Permalink
Cannot retrieve contributors at this time
#!/usr/bin/python | |
import requests | |
from requests.adapters import HTTPAdapter | |
from requests.packages.urllib3.util.retry import Retry | |
import json | |
from pkg_resources import parse_version | |
import argparse | |
import os | |
import subprocess | |
import logging | |
import datetime | |
import time | |
import sys | |
import re | |
parser = argparse.ArgumentParser(description='Syncs images from a public docker registry to a private registry. Use ' | |
'this to populate private registries in a closed off environment. Must be ' | |
'run from a linux host capable of running docker commands which has ' | |
'access both to the internet and the private registry.', | |
epilog='%s \nSample usage: --from=<public-registry-hostname> ' | |
'--to=<registry-to-sync-to> --file=<image-file>', | |
formatter_class=argparse.RawTextHelpFormatter) | |
parser.add_argument('--from', action='store', dest='remote_registry', help='The location of the remote repository', | |
required=True) | |
parser.add_argument('--to', action='store', dest='local_registry', help='The location of the local repository', | |
required=True) | |
parser.add_argument('--file', action='store', dest='json_file', help='A JSON formatted file with the following format:' | |
'{"<tag_type>": {"<namespace>": ["image1", image2"' | |
'image3]}}', required=True) | |
parser.add_argument('--dry-run', action='store_true', dest='dry_run', help='If this flag is present, commands will be' | |
'dumped to stdout instead of run') | |
parser.add_argument('--openshift-version', action='store', dest='ocp_version', | |
help='The version of OpenShift which you want to sync images for') | |
options = parser.parse_args() | |
# Set up the logger | |
todays_date = str(datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d--%H_%M')) | |
logging.basicConfig(level=logging.DEBUG, | |
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', | |
datefmt='%m-%d %H:%M', | |
filename='/tmp/docker_registry_%s.log' % todays_date, | |
filemode='w') | |
# define a Handler which writes INFO messages or higher to the sys.stderr | |
console = logging.StreamHandler() | |
console.setLevel(logging.INFO) | |
# set a format which is simpler for console use | |
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') | |
console.setFormatter(formatter) | |
logging.getLogger('').addHandler(console) | |
# Set the default release version incase unspecified | |
release_version = '3.7' | |
# Set the default registry auth file location | |
docker_auth_config_json = "/root/.docker/config.json" | |
if options.ocp_version is not None: | |
release_version = options.ocp_version | |
retrieve_v_tags_from_redhat_list = [] | |
retrieve_non_v_tags_from_redhat_list = [] | |
latest_tag_list = [] | |
failed_images = [] | |
# Retry if upstream CDN return listed 50x response. | |
def session_with_retry( | |
retries=3, | |
backoff_factor=0.3, | |
status_forcelist=(500, 502, 504), | |
session=None, | |
): | |
session = session or requests.Session() | |
retry = Retry( | |
total=retries, | |
read=retries, | |
connect=retries, | |
backoff_factor=backoff_factor, | |
status_forcelist=status_forcelist, | |
) | |
adapter = HTTPAdapter(max_retries=retry) | |
session.mount('http://', adapter) | |
session.mount('https://', adapter) | |
return session | |
# Take first value of api list generated and attempt a GET to see if upstream server challenge us back for auth. | |
def get_registry_auth_mechanism(url_list): | |
try: | |
auth_session = requests.Session() | |
test_api_url = url_list[1] | |
logging.debug("Test if authentication mechanism configured for URL: %s" % test_api_url) | |
remote_auth_session = session_with_retry(session=auth_session).get(test_api_url) | |
remote_auth_resp_code = remote_auth_session.status_code | |
if remote_auth_resp_code == 200: | |
return False | |
elif remote_auth_resp_code == 401: | |
# Get header for auth challenge, transform and return back to caller. | |
# This check fully conform with https://tools.ietf.org/html/rfc7235#page-7 | |
auth_challenge_realm = remote_auth_session.headers['WWW-Authenticate'] | |
match = re.compile(r"=(.*)") | |
header_match = match.search(auth_challenge_realm) | |
challenge_url_join = header_match.group(1).replace('",', "?") | |
challenge_url = challenge_url_join.replace('"', '') | |
logging.info("Received WWW-Authenticate challenge at: %s" % challenge_url) | |
return challenge_url | |
except (ValueError) as e: | |
print("Upstream registry malfunction when getting %s with HTTP response: %s" % (test_api_url, | |
remote_auth_resp_code)) | |
print("Terminating program, please retry again...") | |
raise e | |
# Authenticate with docker v2 compatible registry, read from config.json as auth token input, | |
# authenticate to the challenge server, get access token and return it to the caller. | |
def get_registry_access_token(registry_name, challenge_url, remote_registry_auth_json_file=None): | |
if remote_registry_auth_json_file: | |
auth_config_location = remote_registry_auth_json_file | |
else: | |
from os.path import expanduser | |
user_home_dir = expanduser("~") | |
auth_config_location = "%s/.docker/config.json" % user_home_dir | |
if not auth_config_location: | |
print(auth_config_location) | |
print("Unable to find and read config.json for authentication. " | |
"Did you performed 'docker login' or supply auth file?") | |
sys.exit(1) | |
else: | |
with open(auth_config_location) as auth_data: | |
reg_auth_json_data = json.load(auth_data) | |
reg_upstream_auth_user_token = reg_auth_json_data['auths'][registry_name]['auth'] | |
try: | |
auth_session = requests.Session() | |
auth_header = {'Authorization': 'Basic %s' % reg_upstream_auth_user_token} | |
get_auth_token = auth_session.get(challenge_url, headers=auth_header) | |
parse_get_resp = json.loads(get_auth_token.text) | |
reg_upstream_access_token = parse_get_resp['access_token'] | |
return reg_upstream_access_token | |
except ValueError as e: | |
return e | |
def generate_url_list(dictionary_key, list_to_populate, remote_registry): | |
for namespace in config_file_dict[dictionary_key]: | |
for image in config_file_dict[dictionary_key][namespace]: | |
docker_json_link = "https://%s/v2/%s/%s/tags/list" % (remote_registry, namespace, image) | |
list_to_populate.append(docker_json_link) | |
# TODO: failed_image_list is not being called in this function. Might be removed in future. | |
def get_latest_tag_from_api(url_list, tag_list, failed_image_list, version_type=None, registry_access_token=None): | |
session = requests.Session() | |
for url in url_list: | |
logging.info("Processing tags for: %s" % url) | |
# If registry_access_token supplied add authorization bearer header to GET request. | |
if registry_access_token: | |
remote_access_auth_header = {'Authorization': 'Bearer %s' % registry_access_token} | |
remote_registry_resp = session_with_retry(session=session).get(url, headers=remote_access_auth_header) | |
else: | |
remote_registry_resp = session_with_retry(session=session).get(url) | |
try: | |
# The object is returned as a string so it needs to be converted to a json object | |
image_tag_dictionary = json.loads(remote_registry_resp.text) | |
except (requests.Except, ValueError) as e: | |
logging.error("ERROR: Unable to parse response from registry") | |
logging.error(" URL: %s" % url) | |
logging.error(" Response Code: %s" % remote_registry_resp.status_code) | |
logging.error(" Response: %s" % remote_registry_resp.text) | |
raise e | |
# Get the latest version for a given release | |
latest_tag = '' | |
image_name = image_tag_dictionary['name'] | |
for tag in image_tag_dictionary['tags']: | |
# check to see if there is a 'v' in the version tag: | |
if tag.startswith('v'): | |
# Ensures that a valid version is being parsed and searched for | |
# release based on a two series release (i.e. 3.9 or 3.10) | |
temp_tag = tag.split('.') | |
req_tag = ".".join(temp_tag[:2]) | |
else: | |
req_tag = '' | |
if release_version in req_tag or 'openshift' not in url: | |
# if release_version in tag[:splice_position] or not 'openshift' in url: | |
# There may be a better way of getting the highest tag for a release | |
# but the list may potentially have a higher release version than what you are looking for | |
if parse_version(tag) > parse_version(latest_tag): | |
if version_type is not None: | |
if "v" in tag: | |
pass | |
else: | |
latest_tag = tag | |
else: | |
latest_tag = tag | |
# We want to remove everything after the hyphen because we don't care about release versions | |
latest_tag_minus_hyphen = latest_tag.split('-')[0] | |
# If the tag has successfully removed a hyphen, it will be unicode, otherwise it will be a string | |
if type(latest_tag_minus_hyphen) is not unicode: | |
logging.error("Unable to match the version for image: %s" % image_name) | |
logging.error("Are you sure that the version exists in the RedHat registry?") | |
logging.info("Attempting to pull image tag 'latest' instead") | |
# insted of failing we will try to pull the tag 'latest' | |
# failed_image_list.append(image_name) | |
tag_list.append("%s:%s" % (image_name, 'latest')) | |
else: | |
tag_list.append("%s:%s" % (image_name, latest_tag_minus_hyphen)) | |
# If package is an rhgs3 package grab the version and 'latest', discovered trying to deploy 3.10 disconnected | |
if 'rhgs3/' in image_name: | |
tag_list.append("%s:%s" % (image_name, 'latest')) | |
def generate_realtime_output(args): | |
response = subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True, stderr=subprocess.STDOUT) | |
for stdout_line in iter(response.stdout.readline, ""): | |
yield stdout_line.strip() | |
response.stdout.close() | |
return_code = response.wait() | |
if return_code: | |
logging.error("There appears to be a problem contacting %s" % args[2]) | |
logging.error("Skipping...") | |
yield False | |
return | |
def dry_run_print_docker_commands(remote_registry, local_registry, image_name): | |
print("docker pull %s/%s" % (remote_registry, image_name)) | |
print("docker tag %s/%s %s/%s" % (remote_registry, image_name, local_registry, image_name)) | |
print("docker push %s/%s" % (local_registry, image_name)) | |
print("") | |
def pull_images(remote_registry, image_name): | |
logging.info("Pulling %s/%s" % (remote_registry, image_name.strip())) | |
exit_with_error = False | |
for pulled_image in generate_realtime_output(["docker", "pull", "%s/%s" % (remote_registry, | |
image_name)]): | |
if pulled_image: | |
logging.info(pulled_image), | |
else: | |
exit_with_error = True | |
break | |
return exit_with_error | |
def tag_images(remote_registry, local_registry, image_name): | |
os.popen("docker tag %s/%s %s/%s" % (remote_registry, image_name, local_registry, | |
image_name)).read() | |
def push_images(local_registry, image_name): | |
for pushed_image in generate_realtime_output(["docker", "push", "%s/%s" % (local_registry, | |
image_name)]): | |
logging.info(pushed_image), | |
config_file = options.json_file | |
with open(config_file) as json_data: | |
config_file_dict = json.load(json_data) | |
generate_url_list('core_components', retrieve_v_tags_from_redhat_list, options.remote_registry) | |
generate_url_list('hosted_components', retrieve_non_v_tags_from_redhat_list, options.remote_registry) | |
auth_challenge_url = get_registry_auth_mechanism(retrieve_v_tags_from_redhat_list) | |
if auth_challenge_url is False: | |
logging.info("Registry did not return challenge, continue...") | |
get_latest_tag_from_api(retrieve_v_tags_from_redhat_list, latest_tag_list, failed_images, | |
registry_access_token=None) | |
get_latest_tag_from_api(retrieve_non_v_tags_from_redhat_list, latest_tag_list, failed_images, | |
registry_access_token=None) | |
else: | |
logging.info("Registry asking for challenge, authenticating (ensure you already run docker login!)...") | |
reg_access_token = get_registry_access_token(options.remote_registry, auth_challenge_url, | |
remote_registry_auth_json_file=docker_auth_config_json) | |
get_latest_tag_from_api(retrieve_v_tags_from_redhat_list, latest_tag_list, failed_images, | |
registry_access_token=reg_access_token) | |
get_latest_tag_from_api(retrieve_non_v_tags_from_redhat_list, latest_tag_list, failed_images, | |
registry_access_token=reg_access_token) | |
total_number_of_images_to_download = len(latest_tag_list) | |
counter = 1 | |
logging.info("") | |
logging.info("Total images to download: %s" % total_number_of_images_to_download) | |
# | |
# Generate initial array list if local registry is a tar file | |
# | |
if options.local_registry == 'tar': | |
cmd = ['docker', 'save', '-o', 'ose3-images.tar'] | |
for namespace_and_image in latest_tag_list: | |
if options.dry_run: | |
logging.info("Dry run mode activated. Docker commands were outputted to the screen") | |
dry_run_print_docker_commands(options.remote_registry, options.local_registry, namespace_and_image) | |
elif options.local_registry == 'tar': | |
# create export images for a tar | |
if not pull_images(options.remote_registry, namespace_and_image): | |
cmd.append(options.remote_registry + '/' + namespace_and_image) | |
else: | |
logging.info("") | |
logging.info("Downloading image %s/%s" % (counter, total_number_of_images_to_download)) | |
counter += 1 | |
# If the image pull fails skip the tagging and pushing steps | |
if not pull_images(options.remote_registry, namespace_and_image): | |
logging.info("Tagging for this registry: %s" % options.local_registry) | |
tag_images(options.remote_registry, options.local_registry, namespace_and_image) | |
logging.info("Pushing into the local registry...") | |
push_images(options.local_registry, namespace_and_image) | |
## | |
# create tar file of images | |
# | |
if options.local_registry == 'tar': | |
for output in generate_realtime_output(cmd): | |
logging.info(output), | |
if failed_images: | |
number_of_failures = len(failed_images) | |
number_of_images_attempted = total_number_of_images_to_download + number_of_failures | |
logging.warning("") | |
logging.warning("%s/%s failed to download version requested fell back to 'latest': %s" | |
% (number_of_failures, number_of_images_attempted, failed_images)) |