Skip to content
Permalink
master
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
 
 
Cannot retrieve contributors at this time
executable file 333 lines (285 sloc) 15.4 KB
#!/usr/bin/python
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
import json
from pkg_resources import parse_version
import argparse
import os
import subprocess
import logging
import datetime
import time
import sys
import re
parser = argparse.ArgumentParser(description='Syncs images from a public docker registry to a private registry. Use '
'this to populate private registries in a closed off environment. Must be '
'run from a linux host capable of running docker commands which has '
'access both to the internet and the private registry.',
epilog='%s \nSample usage: --from=<public-registry-hostname> '
'--to=<registry-to-sync-to> --file=<image-file>',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--from', action='store', dest='remote_registry', help='The location of the remote repository',
required=True)
parser.add_argument('--to', action='store', dest='local_registry', help='The location of the local repository',
required=True)
parser.add_argument('--file', action='store', dest='json_file', help='A JSON formatted file with the following format:'
'{"<tag_type>": {"<namespace>": ["image1", image2"'
'image3]}}', required=True)
parser.add_argument('--dry-run', action='store_true', dest='dry_run', help='If this flag is present, commands will be'
'dumped to stdout instead of run')
parser.add_argument('--openshift-version', action='store', dest='ocp_version',
help='The version of OpenShift which you want to sync images for')
options = parser.parse_args()
# Set up the logger
todays_date = str(datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d--%H_%M'))
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='/tmp/docker_registry_%s.log' % todays_date,
filemode='w')
# define a Handler which writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
# Set the default release version incase unspecified
release_version = '3.7'
# Set the default registry auth file location
docker_auth_config_json = "/root/.docker/config.json"
if options.ocp_version is not None:
release_version = options.ocp_version
retrieve_v_tags_from_redhat_list = []
retrieve_non_v_tags_from_redhat_list = []
latest_tag_list = []
failed_images = []
# Retry if upstream CDN return listed 50x response.
def session_with_retry(
retries=3,
backoff_factor=0.3,
status_forcelist=(500, 502, 504),
session=None,
):
session = session or requests.Session()
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
# Take first value of api list generated and attempt a GET to see if upstream server challenge us back for auth.
def get_registry_auth_mechanism(url_list):
try:
auth_session = requests.Session()
test_api_url = url_list[1]
logging.debug("Test if authentication mechanism configured for URL: %s" % test_api_url)
remote_auth_session = session_with_retry(session=auth_session).get(test_api_url)
remote_auth_resp_code = remote_auth_session.status_code
if remote_auth_resp_code == 200:
return False
elif remote_auth_resp_code == 401:
# Get header for auth challenge, transform and return back to caller.
# This check fully conform with https://tools.ietf.org/html/rfc7235#page-7
auth_challenge_realm = remote_auth_session.headers['WWW-Authenticate']
match = re.compile(r"=(.*)")
header_match = match.search(auth_challenge_realm)
challenge_url_join = header_match.group(1).replace('",', "?")
challenge_url = challenge_url_join.replace('"', '')
logging.info("Received WWW-Authenticate challenge at: %s" % challenge_url)
return challenge_url
except (ValueError) as e:
print("Upstream registry malfunction when getting %s with HTTP response: %s" % (test_api_url,
remote_auth_resp_code))
print("Terminating program, please retry again...")
raise e
# Authenticate with docker v2 compatible registry, read from config.json as auth token input,
# authenticate to the challenge server, get access token and return it to the caller.
def get_registry_access_token(registry_name, challenge_url, remote_registry_auth_json_file=None):
if remote_registry_auth_json_file:
auth_config_location = remote_registry_auth_json_file
else:
from os.path import expanduser
user_home_dir = expanduser("~")
auth_config_location = "%s/.docker/config.json" % user_home_dir
if not auth_config_location:
print(auth_config_location)
print("Unable to find and read config.json for authentication. "
"Did you performed 'docker login' or supply auth file?")
sys.exit(1)
else:
with open(auth_config_location) as auth_data:
reg_auth_json_data = json.load(auth_data)
reg_upstream_auth_user_token = reg_auth_json_data['auths'][registry_name]['auth']
try:
auth_session = requests.Session()
auth_header = {'Authorization': 'Basic %s' % reg_upstream_auth_user_token}
get_auth_token = auth_session.get(challenge_url, headers=auth_header)
parse_get_resp = json.loads(get_auth_token.text)
reg_upstream_access_token = parse_get_resp['access_token']
return reg_upstream_access_token
except ValueError as e:
return e
def generate_url_list(dictionary_key, list_to_populate, remote_registry):
for namespace in config_file_dict[dictionary_key]:
for image in config_file_dict[dictionary_key][namespace]:
docker_json_link = "https://%s/v2/%s/%s/tags/list" % (remote_registry, namespace, image)
list_to_populate.append(docker_json_link)
# TODO: failed_image_list is not being called in this function. Might be removed in future.
def get_latest_tag_from_api(url_list, tag_list, failed_image_list, version_type=None, registry_access_token=None):
session = requests.Session()
for url in url_list:
logging.info("Processing tags for: %s" % url)
# If registry_access_token supplied add authorization bearer header to GET request.
if registry_access_token:
remote_access_auth_header = {'Authorization': 'Bearer %s' % registry_access_token}
remote_registry_resp = session_with_retry(session=session).get(url, headers=remote_access_auth_header)
else:
remote_registry_resp = session_with_retry(session=session).get(url)
try:
# The object is returned as a string so it needs to be converted to a json object
image_tag_dictionary = json.loads(remote_registry_resp.text)
except (requests.Except, ValueError) as e:
logging.error("ERROR: Unable to parse response from registry")
logging.error(" URL: %s" % url)
logging.error(" Response Code: %s" % remote_registry_resp.status_code)
logging.error(" Response: %s" % remote_registry_resp.text)
raise e
# Get the latest version for a given release
latest_tag = ''
image_name = image_tag_dictionary['name']
for tag in image_tag_dictionary['tags']:
# check to see if there is a 'v' in the version tag:
if tag.startswith('v'):
# Ensures that a valid version is being parsed and searched for
# release based on a two series release (i.e. 3.9 or 3.10)
temp_tag = tag.split('.')
req_tag = ".".join(temp_tag[:2])
else:
req_tag = ''
if release_version in req_tag or 'openshift' not in url:
# if release_version in tag[:splice_position] or not 'openshift' in url:
# There may be a better way of getting the highest tag for a release
# but the list may potentially have a higher release version than what you are looking for
if parse_version(tag) > parse_version(latest_tag):
if version_type is not None:
if "v" in tag:
pass
else:
latest_tag = tag
else:
latest_tag = tag
# We want to remove everything after the hyphen because we don't care about release versions
latest_tag_minus_hyphen = latest_tag.split('-')[0]
# If the tag has successfully removed a hyphen, it will be unicode, otherwise it will be a string
if type(latest_tag_minus_hyphen) is not unicode:
logging.error("Unable to match the version for image: %s" % image_name)
logging.error("Are you sure that the version exists in the RedHat registry?")
logging.info("Attempting to pull image tag 'latest' instead")
# insted of failing we will try to pull the tag 'latest'
# failed_image_list.append(image_name)
tag_list.append("%s:%s" % (image_name, 'latest'))
else:
tag_list.append("%s:%s" % (image_name, latest_tag_minus_hyphen))
# If package is an rhgs3 package grab the version and 'latest', discovered trying to deploy 3.10 disconnected
if 'rhgs3/' in image_name:
tag_list.append("%s:%s" % (image_name, 'latest'))
def generate_realtime_output(args):
response = subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True, stderr=subprocess.STDOUT)
for stdout_line in iter(response.stdout.readline, ""):
yield stdout_line.strip()
response.stdout.close()
return_code = response.wait()
if return_code:
logging.error("There appears to be a problem contacting %s" % args[2])
logging.error("Skipping...")
yield False
return
def dry_run_print_docker_commands(remote_registry, local_registry, image_name):
print("docker pull %s/%s" % (remote_registry, image_name))
print("docker tag %s/%s %s/%s" % (remote_registry, image_name, local_registry, image_name))
print("docker push %s/%s" % (local_registry, image_name))
print("")
def pull_images(remote_registry, image_name):
logging.info("Pulling %s/%s" % (remote_registry, image_name.strip()))
exit_with_error = False
for pulled_image in generate_realtime_output(["docker", "pull", "%s/%s" % (remote_registry,
image_name)]):
if pulled_image:
logging.info(pulled_image),
else:
exit_with_error = True
break
return exit_with_error
def tag_images(remote_registry, local_registry, image_name):
os.popen("docker tag %s/%s %s/%s" % (remote_registry, image_name, local_registry,
image_name)).read()
def push_images(local_registry, image_name):
for pushed_image in generate_realtime_output(["docker", "push", "%s/%s" % (local_registry,
image_name)]):
logging.info(pushed_image),
config_file = options.json_file
with open(config_file) as json_data:
config_file_dict = json.load(json_data)
generate_url_list('core_components', retrieve_v_tags_from_redhat_list, options.remote_registry)
generate_url_list('hosted_components', retrieve_non_v_tags_from_redhat_list, options.remote_registry)
auth_challenge_url = get_registry_auth_mechanism(retrieve_v_tags_from_redhat_list)
if auth_challenge_url is False:
logging.info("Registry did not return challenge, continue...")
get_latest_tag_from_api(retrieve_v_tags_from_redhat_list, latest_tag_list, failed_images,
registry_access_token=None)
get_latest_tag_from_api(retrieve_non_v_tags_from_redhat_list, latest_tag_list, failed_images,
registry_access_token=None)
else:
logging.info("Registry asking for challenge, authenticating (ensure you already run docker login!)...")
reg_access_token = get_registry_access_token(options.remote_registry, auth_challenge_url,
remote_registry_auth_json_file=docker_auth_config_json)
get_latest_tag_from_api(retrieve_v_tags_from_redhat_list, latest_tag_list, failed_images,
registry_access_token=reg_access_token)
get_latest_tag_from_api(retrieve_non_v_tags_from_redhat_list, latest_tag_list, failed_images,
registry_access_token=reg_access_token)
total_number_of_images_to_download = len(latest_tag_list)
counter = 1
logging.info("")
logging.info("Total images to download: %s" % total_number_of_images_to_download)
#
# Generate initial array list if local registry is a tar file
#
if options.local_registry == 'tar':
cmd = ['docker', 'save', '-o', 'ose3-images.tar']
for namespace_and_image in latest_tag_list:
if options.dry_run:
logging.info("Dry run mode activated. Docker commands were outputted to the screen")
dry_run_print_docker_commands(options.remote_registry, options.local_registry, namespace_and_image)
elif options.local_registry == 'tar':
# create export images for a tar
if not pull_images(options.remote_registry, namespace_and_image):
cmd.append(options.remote_registry + '/' + namespace_and_image)
else:
logging.info("")
logging.info("Downloading image %s/%s" % (counter, total_number_of_images_to_download))
counter += 1
# If the image pull fails skip the tagging and pushing steps
if not pull_images(options.remote_registry, namespace_and_image):
logging.info("Tagging for this registry: %s" % options.local_registry)
tag_images(options.remote_registry, options.local_registry, namespace_and_image)
logging.info("Pushing into the local registry...")
push_images(options.local_registry, namespace_and_image)
##
# create tar file of images
#
if options.local_registry == 'tar':
for output in generate_realtime_output(cmd):
logging.info(output),
if failed_images:
number_of_failures = len(failed_images)
number_of_images_attempted = total_number_of_images_to_download + number_of_failures
logging.warning("")
logging.warning("%s/%s failed to download version requested fell back to 'latest': %s"
% (number_of_failures, number_of_images_attempted, failed_images))