From 6805093c880eb038751520cf3374ad0485f0f646 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 30 Apr 2015 12:43:45 +0200 Subject: [PATCH 1/7] Add new utility script to be used to create the fedora-install-X repositories --- utility/mm2_create_install_repo | 148 ++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 utility/mm2_create_install_repo diff --git a/utility/mm2_create_install_repo b/utility/mm2_create_install_repo new file mode 100644 index 000000000..af6d8c171 --- /dev/null +++ b/utility/mm2_create_install_repo @@ -0,0 +1,148 @@ +#!/usr/bin/env python + +""" +This script creates the fedora-install repo for Fedora . + +Basically, it checks the content of the nfs mount, if the directory +contains a lot of files, it takes the 3 most recents ones, stores them in +the DB and they will be used later on to check if the mirrors are up to date. +If the directory does not contain too many files, it will register them all +and thus will check them all on the mirrors. +The threshold is stored in: `short_filelist` and is currently at 10. + +If the script finds a yum or atomic repository (finds a repo data or an atomic +summary file), it will create a repository object (cf `make_repository`) which +is basically a mapping between a yum repo name (ie: Fedora-20-updates) and a +directory (/pub/fedora/linux/updates/....) + +TODO: test IRL +""" + +import logging +import optparse +import os +import re +import sys + +sys.path.insert(0, os.path.join(os.path.dirname( + os.path.abspath(__file__)), '..')) +import mirrormanager2.lib +from mirrormanager2.lib.model import Repository + + +logger = None + +originalCategory = 'Fedora Other' + + #dict(subpath='Workstation/armhfp/os', prefix="fedora-workstation-%s", arch="armhfp"), + #dict(subpath='Workstation/i386/os', prefix="fedora-workstation-%s", arch="i386"), + #dict(subpath='Workstation/x86_64/os', prefix="fedora-workstation-%s", arch='x86_64'), +REPOS = [ + dict(subpath='Server/armhfp/os', prefix="fedora-install-%s", arch="armhfp"), + dict(subpath='Server/i386/os', prefix="fedora-install-%s", arch="i386"), + dict(subpath='Server/x86_64/os', prefix="fedora-install-%s", arch="x86_64"), + ] + #dict(subpath='Server/aarch64/os', prefix="fedora-install-%s", arch="aarch64"), + #dict(subpath='Server/ppc64/os', prefix="fedora-install-%s", arch="ppc64"), + #dict(subpath='Server/ppc64le/os', prefix="fedora-install-%s", arch="ppc64le"), + #dict(subpath='Server/s390x/os', prefix="fedora-install-%s", arch="s390x"), + + + +def add_one_repository(directory, category, version, prefix, arch): + """ Add a repository to the database based on the information provided. + """ + + repo = Repository( + prefix=prefix, + version=version, + arch=arch, + name=directory.name, + category=category, + directory=directory + ) + logger.info( + 'Created Repository(prefix=%s, version=%s, arch=%s, category=%s) ' + '-> Directory %s' % ( + prefix, version.name, arch.name, category.name, directory.name + ) + ) + return repo + + + +def doit(session, version, category, args): + """ Actually add the repositories to the database. """ + + ver = mirrormanager2.lib.get_version_by_id(session, version) + category = mirrormanager2.lib.get_category_by_name(session, category) + + parent = args[0] + for r in REPOS: + prefix = r['prefix'] % version + arch = mirrormanager2.lib.get_arch_by_name(session, r['arch']) + version = mirrormanager2.lib.get_version_by_name_version( + session, p_name=category.product.name, p_version=version) + d = os.path.join(parent, r['subpath']) + directory = mirrormanager2.lib.get_directory_by_name(session, d) + repo = mirrormanager2.lib.get_repo_prefix_arch( + session, prefix=prefix, arch=arch) + + if not repo: + repo = add_one_repository(directory, category, version, prefix, arch) + logger.info("Added pointer (%s, %s) to %s" % (repo.prefix, repo.arch, d)) + + else repo: + old_directory = repo.directory + # do the updates + repo.name = d + repo.directory = directory + repo.version = ver + repo.category = category + repo.arch = arch + logger.info("Moved (%s, %s) from %s to %s" % ( + repo.prefix, repo.arch.name, old_directory.name, d)) + + session.add(repo) + + +def setup_logger(debug): + global logger + logging.basicConfig() + logger = logging.getLogger() + if debug: + logger.setLevel(logging.DEBUG) + + +def main(): + global options + parser = optparse.OptionParser(usage=sys.argv[0] + " [options]") + parser.add_option( + "-c", "--config", + dest="config", default='/etc/mirrormanager/mirrormanager2.cfg', + help="Configuration file to use") + parser.add_option( + "--version", dest="version", default='21', help="Version") + parser.add_option( + "--category", dest="category", default="Fedora Other") + parser.add_option( + "--debug", dest="debug", action="store_true", default=False) + + (options, args) = parser.parse_args() + config = dict() + with open(options.config) as config_file: + exec(compile(config_file.read(), options.config, 'exec'), config) + + session = mirrormanager2.lib.create_session(config['DB_URL']) + + setup_logger(options.debug) + doit(session, options.version, options.category, args) + + session.commit() + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) + From 94ee18134e427680c3f87cd07bc051e49d4e9b79 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 30 Apr 2015 21:15:24 +0200 Subject: [PATCH 2/7] Fix typo, there is no argument to else --- utility/mm2_create_install_repo | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utility/mm2_create_install_repo b/utility/mm2_create_install_repo index af6d8c171..13bd5bc7e 100644 --- a/utility/mm2_create_install_repo +++ b/utility/mm2_create_install_repo @@ -89,10 +89,12 @@ def doit(session, version, category, args): session, prefix=prefix, arch=arch) if not repo: - repo = add_one_repository(directory, category, version, prefix, arch) - logger.info("Added pointer (%s, %s) to %s" % (repo.prefix, repo.arch, d)) + repo = add_one_repository( + directory, category, version, prefix, arch) + logger.info( + "Added pointer (%s, %s) to %s" % (repo.prefix, repo.arch, d)) - else repo: + else: old_directory = repo.directory # do the updates repo.name = d From 7ae3965d699f3d9b9178af26aeda24e3d6ce4458 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 30 Apr 2015 21:22:17 +0200 Subject: [PATCH 3/7] Do not override the variables --- utility/mm2_create_install_repo | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/utility/mm2_create_install_repo b/utility/mm2_create_install_repo index 13bd5bc7e..400688af7 100644 --- a/utility/mm2_create_install_repo +++ b/utility/mm2_create_install_repo @@ -71,18 +71,19 @@ def add_one_repository(directory, category, version, prefix, arch): -def doit(session, version, category, args): +def doit(session, version_name, category_name, args): """ Actually add the repositories to the database. """ - ver = mirrormanager2.lib.get_version_by_id(session, version) - category = mirrormanager2.lib.get_category_by_name(session, category) + ver = mirrormanager2.lib.get_version_by_id(session, version_name) + category = mirrormanager2.lib.get_category_by_name( + session, category_name) parent = args[0] for r in REPOS: - prefix = r['prefix'] % version + prefix = r['prefix'] % version_name arch = mirrormanager2.lib.get_arch_by_name(session, r['arch']) version = mirrormanager2.lib.get_version_by_name_version( - session, p_name=category.product.name, p_version=version) + session, p_name=category.product.name, p_version=version_name) d = os.path.join(parent, r['subpath']) directory = mirrormanager2.lib.get_directory_by_name(session, d) repo = mirrormanager2.lib.get_repo_prefix_arch( From 1d34fd337479aac4a30176c6cd7b94888e063acb Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 30 Apr 2015 21:23:25 +0200 Subject: [PATCH 4/7] Error out if the user provides an invalid version or category --- utility/mm2_create_install_repo | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utility/mm2_create_install_repo b/utility/mm2_create_install_repo index 400688af7..d5016f08a 100644 --- a/utility/mm2_create_install_repo +++ b/utility/mm2_create_install_repo @@ -75,8 +75,14 @@ def doit(session, version_name, category_name, args): """ Actually add the repositories to the database. """ ver = mirrormanager2.lib.get_version_by_id(session, version_name) + if not ver: + print 'No such version found: %s' % version_name + return 1 category = mirrormanager2.lib.get_category_by_name( session, category_name) + if not category: + print 'No such category found: %s' % category_name + return 1 parent = args[0] for r in REPOS: From 5c865395b2cc53d9a423cfdc9249e2b709ca35c8 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 30 Apr 2015 21:25:44 +0200 Subject: [PATCH 5/7] Apparently it expects the arch name, not the object --- utility/mm2_create_install_repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utility/mm2_create_install_repo b/utility/mm2_create_install_repo index d5016f08a..2713255d0 100644 --- a/utility/mm2_create_install_repo +++ b/utility/mm2_create_install_repo @@ -93,7 +93,7 @@ def doit(session, version_name, category_name, args): d = os.path.join(parent, r['subpath']) directory = mirrormanager2.lib.get_directory_by_name(session, d) repo = mirrormanager2.lib.get_repo_prefix_arch( - session, prefix=prefix, arch=arch) + session, prefix=prefix, arch=arch.name) if not repo: repo = add_one_repository( From 8a222c3edfa3ada93bbaac276fbbb6e4120eaa77 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 30 Apr 2015 21:50:06 +0200 Subject: [PATCH 6/7] Allow to update a directory that did not exist Sometime when we run the script there is an error and that leads to the data in the DB to be incomplete (for example: missing a Directory), with this change, we can re-run the script and we will update what was wrong fine. --- utility/mm2_create_install_repo | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utility/mm2_create_install_repo b/utility/mm2_create_install_repo index 2713255d0..12dfbb682 100644 --- a/utility/mm2_create_install_repo +++ b/utility/mm2_create_install_repo @@ -109,8 +109,9 @@ def doit(session, version_name, category_name, args): repo.version = ver repo.category = category repo.arch = arch + old_dir = old_directory.name if old_directory else None logger.info("Moved (%s, %s) from %s to %s" % ( - repo.prefix, repo.arch.name, old_directory.name, d)) + repo.prefix, repo.arch.name, old_dir, d)) session.add(repo) From 94d419929c7e0729b916e9a5c3ccd94233f5df44 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 30 Apr 2015 22:01:27 +0200 Subject: [PATCH 7/7] Enforce providing at least one positional argument to the script and give an example --- utility/mm2_create_install_repo | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utility/mm2_create_install_repo b/utility/mm2_create_install_repo index 12dfbb682..4ba282e4a 100644 --- a/utility/mm2_create_install_repo +++ b/utility/mm2_create_install_repo @@ -73,6 +73,10 @@ def add_one_repository(directory, category, version, prefix, arch): def doit(session, version_name, category_name, args): """ Actually add the repositories to the database. """ + if not args: + print 'Must simply the path to the base of the install repo' + print 'for example: mm2_create_install_repo --version=21 '\ + '--category="Fedora Linux" pub/fedora/linux/releases/21/' ver = mirrormanager2.lib.get_version_by_id(session, version_name) if not ver: