Permalink
Cannot retrieve contributors at this time
Name already in use
A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
openshift-toolkit/disconnected_registry/docker-registry-sync.py
Go to fileThis commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
executable file
333 lines (285 sloc)
15.4 KB
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python | |
import requests | |
from requests.adapters import HTTPAdapter | |
from requests.packages.urllib3.util.retry import Retry | |
import json | |
from pkg_resources import parse_version | |
import argparse | |
import os | |
import subprocess | |
import logging | |
import datetime | |
import time | |
import sys | |
import re | |
parser = argparse.ArgumentParser(description='Syncs images from a public docker registry to a private registry. Use ' | |
'this to populate private registries in a closed off environment. Must be ' | |
'run from a linux host capable of running docker commands which has ' | |
'access both to the internet and the private registry.', | |
epilog='%s \nSample usage: --from=<public-registry-hostname> ' | |
'--to=<registry-to-sync-to> --file=<image-file>', | |
formatter_class=argparse.RawTextHelpFormatter) | |
parser.add_argument('--from', action='store', dest='remote_registry', help='The location of the remote repository', | |
required=True) | |
parser.add_argument('--to', action='store', dest='local_registry', help='The location of the local repository', | |
required=True) | |
parser.add_argument('--file', action='store', dest='json_file', help='A JSON formatted file with the following format:' | |
'{"<tag_type>": {"<namespace>": ["image1", image2"' | |
'image3]}}', required=True) | |
parser.add_argument('--dry-run', action='store_true', dest='dry_run', help='If this flag is present, commands will be' | |
'dumped to stdout instead of run') | |
parser.add_argument('--openshift-version', action='store', dest='ocp_version', | |
help='The version of OpenShift which you want to sync images for') | |
options = parser.parse_args() | |
# Set up the logger | |
todays_date = str(datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d--%H_%M')) | |
logging.basicConfig(level=logging.DEBUG, | |
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', | |
datefmt='%m-%d %H:%M', | |
filename='/tmp/docker_registry_%s.log' % todays_date, | |
filemode='w') | |
# define a Handler which writes INFO messages or higher to the sys.stderr | |
console = logging.StreamHandler() | |
console.setLevel(logging.INFO) | |
# set a format which is simpler for console use | |
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') | |
console.setFormatter(formatter) | |
logging.getLogger('').addHandler(console) | |
# Set the default release version incase unspecified | |
release_version = '3.7' | |
# Set the default registry auth file location | |
docker_auth_config_json = "/root/.docker/config.json" | |
if options.ocp_version is not None: | |
release_version = options.ocp_version | |
retrieve_v_tags_from_redhat_list = [] | |
retrieve_non_v_tags_from_redhat_list = [] | |
latest_tag_list = [] | |
failed_images = [] | |
# Retry if upstream CDN return listed 50x response. | |
def session_with_retry( | |
retries=3, | |
backoff_factor=0.3, | |
status_forcelist=(500, 502, 504), | |
session=None, | |
): | |
session = session or requests.Session() | |
retry = Retry( | |
total=retries, | |
read=retries, | |
connect=retries, | |
backoff_factor=backoff_factor, | |
status_forcelist=status_forcelist, | |
) | |
adapter = HTTPAdapter(max_retries=retry) | |
session.mount('http://', adapter) | |
session.mount('https://', adapter) | |
return session | |
# Take first value of api list generated and attempt a GET to see if upstream server challenge us back for auth. | |
def get_registry_auth_mechanism(url_list): | |
try: | |
auth_session = requests.Session() | |
test_api_url = url_list[1] | |
logging.debug("Test if authentication mechanism configured for URL: %s" % test_api_url) | |
remote_auth_session = session_with_retry(session=auth_session).get(test_api_url) | |
remote_auth_resp_code = remote_auth_session.status_code | |
if remote_auth_resp_code == 200: | |
return False | |
elif remote_auth_resp_code == 401: | |
# Get header for auth challenge, transform and return back to caller. | |
# This check fully conform with https://tools.ietf.org/html/rfc7235#page-7 | |
auth_challenge_realm = remote_auth_session.headers['WWW-Authenticate'] | |
match = re.compile(r"=(.*)") | |
header_match = match.search(auth_challenge_realm) | |
challenge_url_join = header_match.group(1).replace('",', "?") | |
challenge_url = challenge_url_join.replace('"', '') | |
logging.info("Received WWW-Authenticate challenge at: %s" % challenge_url) | |
return challenge_url | |
except (ValueError) as e: | |
print("Upstream registry malfunction when getting %s with HTTP response: %s" % (test_api_url, | |
remote_auth_resp_code)) | |
print("Terminating program, please retry again...") | |
raise e | |
# Authenticate with docker v2 compatible registry, read from config.json as auth token input, | |
# authenticate to the challenge server, get access token and return it to the caller. | |
def get_registry_access_token(registry_name, challenge_url, remote_registry_auth_json_file=None): | |
if remote_registry_auth_json_file: | |
auth_config_location = remote_registry_auth_json_file | |
else: | |
from os.path import expanduser | |
user_home_dir = expanduser("~") | |
auth_config_location = "%s/.docker/config.json" % user_home_dir | |
if not auth_config_location: | |
print(auth_config_location) | |
print("Unable to find and read config.json for authentication. " | |
"Did you performed 'docker login' or supply auth file?") | |
sys.exit(1) | |
else: | |
with open(auth_config_location) as auth_data: | |
reg_auth_json_data = json.load(auth_data) | |
reg_upstream_auth_user_token = reg_auth_json_data['auths'][registry_name]['auth'] | |
try: | |
auth_session = requests.Session() | |
auth_header = {'Authorization': 'Basic %s' % reg_upstream_auth_user_token} | |
get_auth_token = auth_session.get(challenge_url, headers=auth_header) | |
parse_get_resp = json.loads(get_auth_token.text) | |
reg_upstream_access_token = parse_get_resp['access_token'] | |
return reg_upstream_access_token | |
except ValueError as e: | |
return e | |
def generate_url_list(dictionary_key, list_to_populate, remote_registry): | |
for namespace in config_file_dict[dictionary_key]: | |
for image in config_file_dict[dictionary_key][namespace]: | |
docker_json_link = "https://%s/v2/%s/%s/tags/list" % (remote_registry, namespace, image) | |
list_to_populate.append(docker_json_link) | |
# TODO: failed_image_list is not being called in this function. Might be removed in future. | |
def get_latest_tag_from_api(url_list, tag_list, failed_image_list, version_type=None, registry_access_token=None): | |
session = requests.Session() | |
for url in url_list: | |
logging.info("Processing tags for: %s" % url) | |
# If registry_access_token supplied add authorization bearer header to GET request. | |
if registry_access_token: | |
remote_access_auth_header = {'Authorization': 'Bearer %s' % registry_access_token} | |
remote_registry_resp = session_with_retry(session=session).get(url, headers=remote_access_auth_header) | |
else: | |
remote_registry_resp = session_with_retry(session=session).get(url) | |
try: | |
# The object is returned as a string so it needs to be converted to a json object | |
image_tag_dictionary = json.loads(remote_registry_resp.text) | |
except (requests.Except, ValueError) as e: | |
logging.error("ERROR: Unable to parse response from registry") | |
logging.error(" URL: %s" % url) | |
logging.error(" Response Code: %s" % remote_registry_resp.status_code) | |
logging.error(" Response: %s" % remote_registry_resp.text) | |
raise e | |
# Get the latest version for a given release | |
latest_tag = '' | |
image_name = image_tag_dictionary['name'] | |
for tag in image_tag_dictionary['tags']: | |
# check to see if there is a 'v' in the version tag: | |
if tag.startswith('v'): | |
# Ensures that a valid version is being parsed and searched for | |
# release based on a two series release (i.e. 3.9 or 3.10) | |
temp_tag = tag.split('.') | |
req_tag = ".".join(temp_tag[:2]) | |
else: | |
req_tag = '' | |
if release_version in req_tag or 'openshift' not in url: | |
# if release_version in tag[:splice_position] or not 'openshift' in url: | |
# There may be a better way of getting the highest tag for a release | |
# but the list may potentially have a higher release version than what you are looking for | |
if parse_version(tag) > parse_version(latest_tag): | |
if version_type is not None: | |
if "v" in tag: | |
pass | |
else: | |
latest_tag = tag | |
else: | |
latest_tag = tag | |
# We want to remove everything after the hyphen because we don't care about release versions | |
latest_tag_minus_hyphen = latest_tag.split('-')[0] | |
# If the tag has successfully removed a hyphen, it will be unicode, otherwise it will be a string | |
if type(latest_tag_minus_hyphen) is not unicode: | |
logging.error("Unable to match the version for image: %s" % image_name) | |
logging.error("Are you sure that the version exists in the RedHat registry?") | |
logging.info("Attempting to pull image tag 'latest' instead") | |
# insted of failing we will try to pull the tag 'latest' | |
# failed_image_list.append(image_name) | |
tag_list.append("%s:%s" % (image_name, 'latest')) | |
else: | |
tag_list.append("%s:%s" % (image_name, latest_tag_minus_hyphen)) | |
# If package is an rhgs3 package grab the version and 'latest', discovered trying to deploy 3.10 disconnected | |
if 'rhgs3/' in image_name: | |
tag_list.append("%s:%s" % (image_name, 'latest')) | |
def generate_realtime_output(args): | |
response = subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True, stderr=subprocess.STDOUT) | |
for stdout_line in iter(response.stdout.readline, ""): | |
yield stdout_line.strip() | |
response.stdout.close() | |
return_code = response.wait() | |
if return_code: | |
logging.error("There appears to be a problem contacting %s" % args[2]) | |
logging.error("Skipping...") | |
yield False | |
return | |
def dry_run_print_docker_commands(remote_registry, local_registry, image_name): | |
print("docker pull %s/%s" % (remote_registry, image_name)) | |
print("docker tag %s/%s %s/%s" % (remote_registry, image_name, local_registry, image_name)) | |
print("docker push %s/%s" % (local_registry, image_name)) | |
print("") | |
def pull_images(remote_registry, image_name): | |
logging.info("Pulling %s/%s" % (remote_registry, image_name.strip())) | |
exit_with_error = False | |
for pulled_image in generate_realtime_output(["docker", "pull", "%s/%s" % (remote_registry, | |
image_name)]): | |
if pulled_image: | |
logging.info(pulled_image), | |
else: | |
exit_with_error = True | |
break | |
return exit_with_error | |
def tag_images(remote_registry, local_registry, image_name): | |
os.popen("docker tag %s/%s %s/%s" % (remote_registry, image_name, local_registry, | |
image_name)).read() | |
def push_images(local_registry, image_name): | |
for pushed_image in generate_realtime_output(["docker", "push", "%s/%s" % (local_registry, | |
image_name)]): | |
logging.info(pushed_image), | |
config_file = options.json_file | |
with open(config_file) as json_data: | |
config_file_dict = json.load(json_data) | |
generate_url_list('core_components', retrieve_v_tags_from_redhat_list, options.remote_registry) | |
generate_url_list('hosted_components', retrieve_non_v_tags_from_redhat_list, options.remote_registry) | |
auth_challenge_url = get_registry_auth_mechanism(retrieve_v_tags_from_redhat_list) | |
if auth_challenge_url is False: | |
logging.info("Registry did not return challenge, continue...") | |
get_latest_tag_from_api(retrieve_v_tags_from_redhat_list, latest_tag_list, failed_images, | |
registry_access_token=None) | |
get_latest_tag_from_api(retrieve_non_v_tags_from_redhat_list, latest_tag_list, failed_images, | |
registry_access_token=None) | |
else: | |
logging.info("Registry asking for challenge, authenticating (ensure you already run docker login!)...") | |
reg_access_token = get_registry_access_token(options.remote_registry, auth_challenge_url, | |
remote_registry_auth_json_file=docker_auth_config_json) | |
get_latest_tag_from_api(retrieve_v_tags_from_redhat_list, latest_tag_list, failed_images, | |
registry_access_token=reg_access_token) | |
get_latest_tag_from_api(retrieve_non_v_tags_from_redhat_list, latest_tag_list, failed_images, | |
registry_access_token=reg_access_token) | |
total_number_of_images_to_download = len(latest_tag_list) | |
counter = 1 | |
logging.info("") | |
logging.info("Total images to download: %s" % total_number_of_images_to_download) | |
# | |
# Generate initial array list if local registry is a tar file | |
# | |
if options.local_registry == 'tar': | |
cmd = ['docker', 'save', '-o', 'ose3-images.tar'] | |
for namespace_and_image in latest_tag_list: | |
if options.dry_run: | |
logging.info("Dry run mode activated. Docker commands were outputted to the screen") | |
dry_run_print_docker_commands(options.remote_registry, options.local_registry, namespace_and_image) | |
elif options.local_registry == 'tar': | |
# create export images for a tar | |
if not pull_images(options.remote_registry, namespace_and_image): | |
cmd.append(options.remote_registry + '/' + namespace_and_image) | |
else: | |
logging.info("") | |
logging.info("Downloading image %s/%s" % (counter, total_number_of_images_to_download)) | |
counter += 1 | |
# If the image pull fails skip the tagging and pushing steps | |
if not pull_images(options.remote_registry, namespace_and_image): | |
logging.info("Tagging for this registry: %s" % options.local_registry) | |
tag_images(options.remote_registry, options.local_registry, namespace_and_image) | |
logging.info("Pushing into the local registry...") | |
push_images(options.local_registry, namespace_and_image) | |
## | |
# create tar file of images | |
# | |
if options.local_registry == 'tar': | |
for output in generate_realtime_output(cmd): | |
logging.info(output), | |
if failed_images: | |
number_of_failures = len(failed_images) | |
number_of_images_attempted = total_number_of_images_to_download + number_of_failures | |
logging.warning("") | |
logging.warning("%s/%s failed to download version requested fell back to 'latest': %s" | |
% (number_of_failures, number_of_images_attempted, failed_images)) | |