From f9f07585e36fd2a4388c5ca58b72f9f4f5ccedfe Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Wed, 13 Mar 2024 18:03:12 +0800 Subject: [PATCH] update image_syncer (#90) --- .../work/image-syncer/run_jobs.py | 29 +++++++------ .../work/image-syncer/run_sync.py | 42 ++++++++++++------- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/docker_docker_kit/work/image-syncer/run_jobs.py b/docker_docker_kit/work/image-syncer/run_jobs.py index 108564c..f88717b 100644 --- a/docker_docker_kit/work/image-syncer/run_jobs.py +++ b/docker_docker_kit/work/image-syncer/run_jobs.py @@ -1,6 +1,7 @@ +import multiprocessing import os -import json import sys + import yaml import run_sync @@ -23,27 +24,29 @@ def get_job_names_from_yaml(file_path): def main(): - namespace = os.environ.get('IMG_NAMESPACE') - if namespace is None: - print('Using default IMG_NAMESPACE=library !') - namespace = 'library' + img_namespace = os.environ.get('IMG_NAMESPACE', 'library') + registry_url_src = os.environ.get("REGISTRY_URL", 'docker.io') + registry_url_dst = os.environ.get("DOCKER_MIRROR_REGISTRY", None) images = [] job_names = get_job_names_from_yaml('.github/workflows/build-docker.yml') for name in job_names: images.extend(name.split(',')) + list_tasks = [] for image in images: - img = '/'.join([namespace, image]) + img = '/'.join([img_namespace, image]) print("Docker image sync job name found:", img) - configs = run_sync.generate(image=img, tags=None) + configs = run_sync.generate( + image=img, tags=None, source_registry=registry_url_src, target_registries=registry_url_dst + ) for _, c in enumerate(configs): - s_config = json.dumps(c, ensure_ascii=False, sort_keys=True) - print('Config item:', json.dumps(c, ensure_ascii=False, sort_keys=True)) - ret = run_sync.sync_image(cfg=c) - if ret != 0: - return ret - return ret + list_tasks.append(c) + + with multiprocessing.Pool() as pool: + res = pool.map_async(run_sync.sync_image, list_tasks) + ret = sum(res.get()) + return ret if __name__ == '__main__': diff --git a/docker_docker_kit/work/image-syncer/run_sync.py b/docker_docker_kit/work/image-syncer/run_sync.py index 1585680..68c89cf 100644 --- a/docker_docker_kit/work/image-syncer/run_sync.py +++ b/docker_docker_kit/work/image-syncer/run_sync.py @@ -6,12 +6,12 @@ import tempfile -def generate(image: str, target_registries: list = None, tags: list = None, target_image: str = None): +def generate(image: str, source_registry: str = None, target_registries: list = None, tags: list = None): """Generate a config item which will be used by `image-syncer`.""" - uname = os.environ.get('DOCKER_MIRROR_REGISTRY_USERNAME', None) - passwd = os.environ.get('DOCKER_MIRROR_REGISTRY_PASSWORD', None) + uname_mirror = os.environ.get('DOCKER_MIRROR_REGISTRY_USERNAME', None) + passwd_mirror = os.environ.get('DOCKER_MIRROR_REGISTRY_PASSWORD', None) - if uname is None or passwd is None: + if uname_mirror is None or passwd_mirror is None: print('ENV variable required: DOCKER_MIRROR_REGISTRY_USERNAME and DOCKER_MIRROR_REGISTRY_PASSWORD !') sys.exit(-2) @@ -20,16 +20,27 @@ def generate(image: str, target_registries: list = None, tags: list = None, targ destinations = ['cn-beijing', 'cn-hangzhou'] target_registries = ['registry.%s.aliyuncs.com' % i for i in destinations] - for dest in target_registries: - src = "%s:%s" % (image, tags) if tags is not None else image - yield { + for target_registry in target_registries: + img_src_tag = '%s:%s' % (image, tags) if tags is not None else image + img_src: str = "%s/%s" % (source_registry, img_src_tag) + img_dst: str = "%s/%s" % (target_registry, image) + + c = { 'auth': { - dest: {"username": uname, "password": passwd} + target_registry: {"username": uname_mirror, "password": passwd_mirror} }, - 'images': { - src: "%s/%s" % (dest, target_image or image) - } + 'images': {img_src: img_dst} } + if source_registry is not None: + uname_source = os.environ.get('DOCKER_REGISTRY_USERNAME', None) + passwd_source = os.environ.get('DOCKER_REGISTRY_PASSWORD', None) + if uname_source is None or passwd_source is None: + print('ENV variable required: DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD !') + sys.exit(-2) + c['auth'].update({source_registry: { + "username": uname_source, "password": passwd_source} + }) + yield c def sync_image(cfg: dict): @@ -50,13 +61,14 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('img', type=str, help='Source image, with or without tag') parser.add_argument('--tags', type=str, action='extend', nargs='*', help='Tags to sync, optional.') - parser.add_argument('--dest-image', type=str, help='Target image name, with our without tag') - parser.add_argument('--dest-registry', type=str, action='extend', nargs='*', help='tTarget registry URL') + parser.add_argument('--source-registry', type=str, default='docker.io', help='Target image name, with our without tag') + parser.add_argument('--target-registry', type=str, action='extend', nargs='*', help='Target registry URL') args = parser.parse_args() - configs = generate(image=args.img, tags=args.tags, target_registries=args.dest_registry, target_image=args.dest_image) + configs = generate(image=args.img, tags=args.tags, source_registry=args.target_registry, target_registries=args.dest_registry) + ret = 0 for _, c in enumerate(configs): - ret = sync_image(cfg=c) + ret += sync_image(cfg=c) return ret