From 22c389cb0354356a5ff4dd6d6d9d0d5a2c479fd8 Mon Sep 17 00:00:00 2001 From: Ding-Yi Chen Date: Thu, 16 Aug 2018 17:47:43 +1000 Subject: [PATCH] feat(ZNTA-2729): release RPM in Zanata_Team Repo --- ZanataArgParser.py | 107 ++++++++++++++------ ZanataFunctions.py | 143 ++++++++++++++++---------- ZanataRpm.py | 193 +++++++++++++++++++++++++++++++++++ ZanataRpmRepo.py | 199 +++++++++++++++++++++++++++++++++++++ jenkins/elrepo.jenkinsfile | 99 ++++++++++++++++++ pylintrc | 4 + 6 files changed, 665 insertions(+), 80 deletions(-) create mode 100755 ZanataRpm.py create mode 100755 ZanataRpmRepo.py create mode 100644 jenkins/elrepo.jenkinsfile diff --git a/ZanataArgParser.py b/ZanataArgParser.py index 4b6c162..0d638a6 100755 --- a/ZanataArgParser.py +++ b/ZanataArgParser.py @@ -15,7 +15,7 @@ import re import sys -from argparse import ArgumentParser, ArgumentError +from argparse import ArgumentParser, ArgumentError, RawDescriptionHelpFormatter # Following are for mypy from argparse import Action # noqa: F401 # pylint: disable=W0611 from argparse import Namespace # noqa: F401 # pylint: disable=W0611 @@ -30,16 +30,32 @@ sys.stderr.write("python typing module is not installed" + os.linesep) +class NoSuchMethodError(Exception): + """Method does not exist + + Args: + method_name (str): Name of the method + """ + def __init__(self, method_name): + super(NoSuchMethodError, self).__init__() + self.method_name = method_name + + def __str__(self): + return "No such method: %s" % self.method_name + + class ColoredFormatter(logging.Formatter): """Log colored formated Inspired from KurtJacobson's colored_log.py""" - DEFAULT_COLOR = 37 # white - MAPPING = { - 'DEBUG': DEFAULT_COLOR, - 'INFO': 36, # cyan - 'WARNING': 33, # yellow - 'ERROR': 31, # red - 'CRITICAL': 41} # white on red bg + # Background ASCII color + bg = os.getenv("LOGGING_BG_COLOR", 40) # Default black background + + COLOR_MAPPING = { + 'DEBUG': [os.getenv("LOGGING_DEBUG_COLOR", 37), bg], # white + 'INFO': [os.getenv("LOGGING_INFO_COLOR", 36), bg], # cyan + 'WARNING': [os.getenv("LOGGING_WARNING_COLOR", 33), bg], # yellow + 'ERROR': [os.getenv("LOGGING_ERROR_COLOR", 31), bg], # red + 'CRITICAL': [37, 41]} # white on red bg PREFIX = '\033[' SUFFIX = '\033[0m' @@ -48,19 +64,17 @@ def __init__(self, patern): logging.Formatter.__init__(self, patern) @staticmethod - def _color(color_id, content): - return "\033[%dm%s\033[0m" % (color_id, content) + def _color(colors, content): + return "\033[%d;%dm%s\033[0m" % (colors[0], colors[1], content) def format(self, record): - color_id = ColoredFormatter.MAPPING.get( - record.levelname, ColoredFormatter.DEFAULT_COLOR) - record.levelname = ColoredFormatter._color( - color_id, record.levelname) - record.message = ColoredFormatter._color( - color_id, record.getMessage()) + colors = ColoredFormatter.COLOR_MAPPING.get( + record.levelname, ColoredFormatter.COLOR_MAPPING['DEBUG']) + record.levelname = ColoredFormatter._color(colors, record.levelname) + record.message = ColoredFormatter._color(colors, record.getMessage()) if self.usesTime(): record.asctime = ColoredFormatter._color( - color_id, self.formatTime(record, self.datefmt)) + colors, self.formatTime(record, self.datefmt)) try: s = self._fmt % record.__dict__ except UnicodeDecodeError as e: @@ -93,10 +107,30 @@ def format(self, record): class ZanataArgParser(ArgumentParser): - """Zanata Argument Parser""" + """Zanata Argument Parser that support sub-commands and environment + + Examples: + >>> import ZanataArgParser + >>> parser = ZanataArgParser.ZanataArgParser('my-prog') + >>> parser.add_common_argument('-b', '--branch', default='master') + >>> parser.add_sub_command('list', None, None) + >>> args = parser.parse_all(['list', '-b', 'release']) + >>> print(args.sub_command) + list + >>> print(args.branch) + release + >>> args2 = parser.parse_all(['list']) + >>> print(args2.branch) + master + """ + def __init__(self, *args, **kwargs): # type: (Any, Any) -> None - super(ZanataArgParser, self).__init__(*args, **kwargs) + # Ignore mypy "ArgumentParser" gets multiple values for keyword + # argument "formatter_class" + # See https://github.com/python/mypy/issues/1028 + super(ZanataArgParser, self).__init__( + *args, formatter_class=RawDescriptionHelpFormatter, **kwargs) self.env_def = {} # type: Dict[str, dict] self.parent_parser = ArgumentParser(add_help=False) self.add_argument( @@ -105,7 +139,7 @@ def __init__(self, *args, **kwargs): help='Valid values: %s' % 'DEBUG, INFO, WARNING, ERROR, CRITICAL, NONE') - self.sub_parsers = None + self.sub_parsers = None # type: _SubParsersAction self.sub_command_obj_dict = {} # type: Dict[str, Any] def add_common_argument(self, *args, **kwargs): @@ -117,7 +151,7 @@ def add_common_argument(self, *args, **kwargs): self.parent_parser.add_argument(*args, **kwargs) def add_sub_command(self, name, arguments, obj=None, **kwargs): - # type: (str, List, Any, Any) -> ArgumentParser + # type: (str, List, Any, Any) -> None """Add a sub command Args: @@ -125,9 +159,7 @@ def add_sub_command(self, name, arguments, obj=None, **kwargs): arguments (dict): argments to be passed to argparse.add_argument() obj (Any, optional): Defaults to None. The sub_command is a method of the obj. - - Returns: - [type]: [description] + kwargs (Any, optional): other arguments for add_parser """ if not self.sub_parsers: self.sub_parsers = self.add_subparsers( @@ -153,7 +185,6 @@ def add_sub_command(self, name, arguments, obj=None, **kwargs): else: anonymous_parser.add_argument(*k.split()) anonymous_parser.set_defaults(sub_command=name) - return anonymous_parser def add_env( # pylint: disable=too-many-arguments self, env_name, @@ -181,12 +212,12 @@ def add_env( # pylint: disable=too-many-arguments 'dest': dest, 'sub_commands': sub_commands} - def add_methods_as_subcommands(self, obj, name_pattern='.*'): + def add_methods_as_sub_commands(self, obj, name_pattern='.*'): # type (Any, str) -> None """Add public methods as sub-commands Args: - obj ([type]): Public methods of obj will be used + cls ([type]): Public methods of obj will be used name_pattern (str, optional): Defaults to '.*'. Method name should match the pattern. """ @@ -205,6 +236,11 @@ def add_methods_as_subcommands(self, obj, name_pattern='.*'): if not inspect.ismethod(m_obj) and not inspect.isfunction(m_obj): continue + if name == 'init_from_parsed_args': + # init_from_parsed_args initialize from parsed args. + # No need to be in sub-commands + continue + argspec = inspect.getargspec(m_obj) sub_args = None try: @@ -229,7 +265,9 @@ def add_methods_as_subcommands(self, obj, name_pattern='.*'): name, sub_args, obj, - help=obj.__doc__) + help=re.sub( + "\n.*$", "", m_obj.__doc__, flags=re.MULTILINE), + description=m_obj.__doc__) def has_common_argument(self, option_string=None, dest=None): # type: (str, str) -> bool @@ -359,6 +397,13 @@ def run_sub_command(self, args=None): "sub-command %s is not associated with any object" % args.sub_command) obj = self.sub_command_obj_dict[args.sub_command] + if inspect.isclass(obj): + cls = obj + if not hasattr(cls, 'init_from_parsed_args'): + raise NoSuchMethodError('init_from_parsed_args') + # New an object accordingto args + obj = getattr(cls, 'init_from_parsed_args')(args) + sub_cmd_obj = getattr(obj, args.sub_command) argspec = inspect.getargspec(sub_cmd_obj) arg_values = [] @@ -370,9 +415,15 @@ def run_sub_command(self, args=None): if __name__ == '__main__': + if os.getenv("PY_DOCTEST", "0") == "1": + import doctest + test_result = doctest.testmod() + print(doctest.testmod(), file=sys.stderr) + sys.exit(0 if test_result.failed == 0 else 1) print("Legend of log levels", file=sys.stderr) ZanataArgParser('parser').parse_args(["-v", "DEBUG"]) logging.debug("debug") logging.info("info") logging.warning("warning") logging.error("error") + logging.critical("critical") diff --git a/ZanataFunctions.py b/ZanataFunctions.py index 520c385..bdc3a50 100755 --- a/ZanataFunctions.py +++ b/ZanataFunctions.py @@ -30,7 +30,19 @@ def read_env(filename): # type (str) -> dict - """Read environment variables by sourcing a bash file""" + """Read environment variables by sourcing a bash file + + Args: + filename (str): Bash file to be read + + Returns: + [dict]: Dict whose key is environment variable name, + and value is variable value. + + Examples: + >>> read_env('zanata-env.sh')['EXIT_OK'] + u'0' + """ proc = subprocess.Popen( # nosec [BASH_CMD, '-c', "source %s && set -o posix && set" % (filename)], @@ -43,18 +55,18 @@ def read_env(filename): ZANATA_ENV = read_env(ZANATA_ENV_FILE) if 'WORK_ROOT' in os.environ: - WORK_ROOT = os.getenv('WORK_ROOT') + WORK_ROOT = str(os.environ.get('WORK_ROOT')) elif ZANATA_ENV['WORK_ROOT']: - WORK_ROOT = ZANATA_ENV['WORK_ROOT'] + WORK_ROOT = str(ZANATA_ENV['WORK_ROOT']) else: - WORK_ROOT = os.getcwd + WORK_ROOT = os.getcwd() class CLIException(Exception): """Exception from command line""" def __init__(self, msg, level='ERROR'): - super(CLIException).__init__(type(self)) + super(CLIException, self).__init__(type(self)) self.msg = "[%s] %s" % (level, msg) def __str__(self): @@ -75,16 +87,31 @@ def __init__( # type: (str, str, str, str) -> None self.user = user self.token = token - url_parsed = urlparse.urlparse(url) + + parsed = urlparse.urlsplit(url) + data = list(parsed) + + # replace if user is specify if user: - url_parsed.username = user - if token: - url_parsed.password = token + userrec = user + if token: + userrec += ":" + token + netloc = "%s@%s" % (userrec, parsed.hostname) + data[1] = netloc self.url = url - self.auth_url = urlparse.urlunparse(url_parsed) + self.auth_url = urlparse.urlunparse(data) self.remote = remote + @classmethod + def init_from_parsed_args(cls, args): + """Init from command line arguments""" + kwargs = {} + for k in ['user', 'token', 'url', 'remote']: + if hasattr(args, k): + kwargs[k] = getattr(args, k) + return cls(**kwargs) + @staticmethod def git_check_output(arg_list, **kwargs): """Run git command and return stdout as string @@ -102,7 +129,7 @@ def git_check_output(arg_list, **kwargs): """ cmd_list = [GitHelper.GIT_CMD] + arg_list logging.debug("Running command: %s", " ".join(cmd_list)) - return subprocess.check_output(cmd_list, **kwargs) + return subprocess.check_output(cmd_list, **kwargs).rstrip() @staticmethod def branch_get_current(): @@ -182,6 +209,11 @@ class SshHost(object): SCP_CMD = '/usr/bin/scp' SSH_CMD = '/usr/bin/ssh' + RSYNC_CMD = '/usr/bin/rsync' + RSYNC_OPTIONS = [ + '--cvs-exclude', '--recursive', '--verbose', '--links', + '--update', '--compress', '--exclude', '*.core', '--stats', + '--progress', '--archive', '--keep-dirlinks'] def __init__(self, host, ssh_user=None, identity_file=None): # type (str, str, str) -> None @@ -219,7 +251,7 @@ def add_parser(cls, arg_parser=None): def init_from_parsed_args(cls, args): """Init from command line arguments""" kwargs = {'host': args.host} - for k in ['ssh_user', 'identitity_file']: + for k in ['ssh_user', 'identity_file']: if hasattr(args, k): kwargs[k] = getattr(args, k) return cls(**kwargs) @@ -272,6 +304,30 @@ def scp_to_host( subprocess.check_call(cmd_list) # nosec + def rsync(self, src, dest, options=None): + # type (str, str, List[Str]) -> None + """Run rsync + + Args: + src (str): src file/dir in rsync + dest (str): src file/dir in rsync + options (List[str], optional): Defaults to None. + List of rsync options. + """ + cmd_prefix = [SshHost.RSYNC_CMD] + SshHost.RSYNC_OPTIONS + if self.ssh_user: + ssh_cmd = "ssh -l {} {} {}".format( + self.ssh_user, + "" if not self.identity_file else "-i", + "" if not self.identity_file else self.identity_file) + cmd_prefix += ["-e", ssh_cmd] + + if options: + cmd_prefix += options + full_cmd = cmd_prefix + [src, dest] + logging.debug(" ".join(full_cmd)) + subprocess.check_call(full_cmd) + class UrlHelper(object): """URL helper functions""" @@ -348,16 +404,21 @@ def mkdir_p(directory, mode=0o755): def version_sort(version_list, reverse=False): - """Sort the version + """Sort the version from list - Arguments: - version_list {List[str]} -- List of versions - - Keyword Arguments: - reverse {bool} -- Whether to reverse sort (default: {False}) + Args: + version_list (List[str]): version str in a list + reverse (bool, optional): Defaults to False. Desending sort. Returns: List[str] -- Sorted list of versions + + Examples: + >>> version_list = ['1.0.0', '10.0.0', '2.0.0', '1.0.0-rc-1' ] + >>> version_sort(version_list) + ['1.0.0-rc-1', '1.0.0', '2.0.0', '10.0.0'] + >>> version_sort(version_list, True) + ['10.0.0', '2.0.0', '1.0.0', '1.0.0-rc-1'] """ # Add -zfinal to final releases, so it can be sorted after rc sorted_dirty_version = sorted( @@ -384,47 +445,25 @@ def working_directory(directory): os.chdir(curr_directory) -def _parse(): +def main(): + """Run as command line program""" parser = ZanataArgParser(__file__) - parser.add_sub_command( - 'list-run', None, - help='list runable functions') - parser.add_sub_command( - 'run', - [ - ('func_name', { - 'type': str, 'default': '', - 'help': 'Function name'}), - ('func_args', { - 'type': str, - 'nargs': '*', - 'help': 'Function arguments'})], - help='Run function') + parser.add_methods_as_sub_commands(GitHelper) parser.add_sub_command( 'module-help', None, help='Show Python Module help') - return parser.parse_all() - + args = parser.parse_all() -def _run_as_cli(): - import inspect - args = _parse() if args.sub_command == 'module-help': help(sys.modules[__name__]) - elif args.sub_command == 'list-run': - cmd_list = inspect.getmembers(GitHelper, predicate=inspect.ismethod) - for cmd in cmd_list: - if cmd[0][0] == '_': - continue - print("%s:\n %s\n" % (cmd[0], cmd[1].__doc__)) - print(inspect.getargspec(cmd[1])) - elif args.sub_command == 'run': - if hasattr(GitHelper, args.func_name): - g_helper = GitHelper() - print(getattr(g_helper, args.func_name)(*args.func_args)) - else: - raise CLIException("No known func name %s" % args.func_name) + else: + parser.run_sub_command(args) if __name__ == '__main__': - _run_as_cli() + if os.getenv("PY_DOCTEST", "0") == "1": + import doctest + test_result = doctest.testmod() + print(doctest.testmod(), file=sys.stderr) + sys.exit(0 if test_result.failed == 0 else 1) + main() diff --git a/ZanataRpm.py b/ZanataRpm.py new file mode 100755 index 0000000..a19cecc --- /dev/null +++ b/ZanataRpm.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python2 +# encoding: utf-8 +""" +ZanataRpm -- RPM manipulate + +ZanataRpm mainpulates RPM and spec files, +such as version bump. + +It defines classes_and_methods + +@author: Ding-Yi Chen + +@copyright: 2018 Red Hat Asia Pacific. All rights reserved. + +@license: LGPLv2+ + +@contact: dchen@redhat.com +""" +from __future__ import absolute_import, division, print_function + +import datetime +import locale +import logging +import re +import os +import sys + +from ZanataArgParser import ZanataArgParser # pylint: disable=import-error +from ZanataFunctions import CLIException + +try: + from typing import List, Any # noqa: F401 # pylint: disable=unused-import +except ImportError: + sys.stderr.write("python typing module is not installed" + os.linesep) + +locale.setlocale(locale.LC_ALL, 'C') + + +class RpmSpec(object): + """ + RPM Spec + """ + + # We only interested in these tags + TAGS = ['Name', 'Version', 'Release'] + + def __init__(self, **kwargs): + # type (Any) -> None + """ + Constructor + """ + for v in kwargs: + setattr(self, v, kwargs.get(v)) + self.content = [] + + def parse_spec_tag(self, line): + # type (str) -> None + """Parse the tag value from line if the line looks like + spec tag definition, otherwise do nothing""" + + s = line.rstrip() + + matched = re.match(r"([A-Z][A-Za-z]*):\s*(.+)", s) + if matched: + if matched.group(1) in RpmSpec.TAGS: + + tag = matched.group(1) + if not hasattr(self, tag): + # Only use the first match + setattr(self, tag, matched .group(2)) + return s + + @classmethod + def init_from_file(cls, spec_file): + # type (str) -> None + """Init from existing spec file + + Args: + spec_file (str): RPM spec file + + Raises: + OSError e: File error + + Returns: + RpmSpec: Instance read from spec_file + """ + try: + with open(spec_file, 'r') as in_file: + self = cls() + self.content = [ + self.parse_spec_tag(l) + for l in in_file.readlines()] + except OSError as e: + raise e + return self + + def update_version(self, version): + # type (str) -> bool + """Update to new version + + Args: + version (str): new version to be set + """ + if getattr(self, 'Version') == version: + logging.warning("Spec file is already with version %s", version) + return False + + setattr(self, 'Version', version) + + # Update content + new_content = [] + for line in self.content: + matched = re.match(r"^Version:(\s*)(.+)", line) + if matched: + new_content.append( + "Version:{}{}".format(matched.group(1), version)) + continue + + changelog_matched = re.match("^%changelog", line) + if changelog_matched: + now = datetime.datetime.now().strftime("%a %b %d %Y") + changelog_item = ( + "* {date} {email} {version}-1\n" + "- Upgrade to upstream version {version}\n".format( + date=now, + email=os.getenv( + 'MAINTAINER_EMAIL', + 'noreply@zanata.org'), + version=version)) + new_content.append(line) + new_content.append(changelog_item) + continue + + new_content.append(line) + + setattr(self, 'content', new_content) + return True + + def write_to_file(self, spec_file): + """Write the spec to file + + Args: + spec_file (str): RPM spec file + + Raises: + OSError e: File error + """ + try: + with open(spec_file, 'w') as out_file: + out_file.write(str(self)) + + except OSError as e: + logging.error("Failed to write to %s", spec_file) + raise e + + def __str__(self): + return "\n".join(getattr(self, 'content')) + + +def _parse(): + parser = ZanataArgParser(__file__) + parser.add_sub_command( + 'update-version', + [ + ('--force -f', { + 'action': 'store_true', + 'help': 'Force overwritten'}), + ('spec_file', { + 'type': str, + 'help': 'spec file'}), + ('version', { + 'type': str, + 'help': 'new version'})], + help=RpmSpec.__doc__) + return parser.parse_all() + + +def main(): + """Run as command line program""" + args = _parse() + if args.sub_command == 'help': + help(sys.modules[__name__]) + else: + if args.sub_command == 'update-version': + instance = RpmSpec.init_from_file(args.spec_file) + instance.update_version(args.version) + instance.write_to_file(args.spec_file) + else: + raise CLIException("No known sub command %s" % args.sub_command) + + +if __name__ == '__main__': + main() diff --git a/ZanataRpmRepo.py b/ZanataRpmRepo.py new file mode 100755 index 0000000..cce39db --- /dev/null +++ b/ZanataRpmRepo.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +# encoding: utf-8 +"""ZanataRpmRepo -- Release package in dnf/yum repo + +ZanataRpmRepo builds and Zanata RPM packages and upload +to the remote dnf/yum repository. + +Requires: + * docker + * rsync + * ssh + +@author: Ding-Yi Chen + +@copyright: 2018 Red Hat Asia Pacific. All rights reserved. + +@license: LGPLv2+ + +@contact: dchen@redhat.com +@deffield updated: Updated +""" +from __future__ import absolute_import, division, print_function + +import logging +import os +import subprocess +import sys + +from ZanataArgParser import ZanataArgParser # pylint: disable=E0401 +from ZanataFunctions import GitHelper, SshHost, WORK_ROOT +from ZanataFunctions import mkdir_p, working_directory + +try: + # We need to import 'List' and 'Any' for mypy to work + from typing import List, Any # noqa: F401 # pylint: disable=unused-import +except ImportError: + sys.stderr.write("python typing module is not installed" + os.linesep) + +PROFILE = 0 + +LOCAL_DIR = os.path.join(WORK_ROOT, 'dnf', 'zanata') + + +class RpmRepoHost(SshHost): + """Host that hosts Rpm Repo""" + FEDORAPEOPLE_HOST = 'fedorapeople.org' + + def __init__( # pylint: disable=too-many-arguments + self, host=FEDORAPEOPLE_HOST, + ssh_user=None, identity_file=None, + remote_dir='/srv/repos/Zanata_Team/zanata', + local_dir=LOCAL_DIR): + super(RpmRepoHost, self).__init__(host, ssh_user, identity_file) + self.remote_dir = remote_dir + self.remote_host_dir = "%s:%s" % (self.user_host, self.remote_dir) + self.local_dir = local_dir + + @classmethod + def init_from_parsed_args(cls, args): + """Init from command line arguments""" + setattr(args, 'host', RpmRepoHost.FEDORAPEOPLE_HOST) + return super(RpmRepoHost, cls).init_from_parsed_args(args) + + def pull(self): + # type (str) -> None + """Pull from remote directory + """ + mkdir_p(self.local_dir) + src_dir = os.path.join(self.remote_host_dir, '') + logging.info("Pull from %s to %s", src_dir, self.local_dir) + self.rsync(src_dir, self.local_dir, ['--delete']) + + def update_epel_repos( + self, spec_file, version='auto', dist_versions=None): + """Update all EPEL repositories + + Args: + spec_file (str): RPM spec file + dist_versions (List[str]): Defaults to ["7", "6"]. + List of distrion versions to update. + """ + if not dist_versions: + dist_versions = ["7", "6"] + for dist in dist_versions: + logging.info("Update EL%s repo", dist) + elrepo = ElRepo(dist, self.local_dir) + elrepo.build_and_update(spec_file, version) + + def push(self): + # type (str) -> None + """Push local files to remote directory + """ + src_dir = os.path.join(self.local_dir, '') + logging.info("Push from %s to %s", src_dir, self.remote_host_dir) + self.rsync(src_dir, self.remote_host_dir, ['--delete']) + + def all(self, spec_file, version='auto'): + """Run the full cycle + + Args: + spec_file (str): RPM spec file + version (str, optional): Defaults to 'auto'. New version of the + packages. + dist_versions (List[str]): Defaults to ["7", "6"]. + List of distrion versions to update. + """ + self.pull() + self.update_epel_repos(spec_file, version) + self.push() + + +class ElRepo(object): # pylint: disable=too-few-public-methods + """A dnf/yum repository for Enterprisse Linux (EL) + + Each repository contains exactly one EPEL release (dist). + Repository also contains RPMs for following arch: + x86_64, i386, noarch, src + """ + + def __init__(self, dist_ver, local_dir=LOCAL_DIR): + # type (str) -> None + """New an ElRepo given distribution version + + Args: + dist_ver (str): Distribution version like "7" or "6" + loca_dir (str, optional): Defaults to LOCAL_DIR. Local directory + """ + self.dist_ver = dist_ver + self.local_dir = local_dir + + def build_and_update(self, spec_file, version=None): + # type () -> None + """Build RPM and update dnf/yum repository + + Args: + spec_file (str): RPM spec file + """ + with working_directory(self.local_dir): + volume_name = "zanata-el-%s-repo" % self.dist_ver + vols = subprocess.check_output([ + 'docker', 'volume', 'ls', '-q']).rstrip().split('\n') + if volume_name not in vols: + subprocess.check_call([ + 'docker', 'volume', 'create', '--name', volume_name]) + + docker_run_cmd = [ + "docker", "run", "--rm", "--name", + "zanata-el-{}-builder".format(self.dist_ver), + "-v", "{}:/repo:Z".format(volume_name), + "-v", "/tmp:/rpmbuild/SOURCES:Z", + "-v", "{}:/repo_host_dir:Z".format(self.local_dir), + "-v", "{}:/output_dir:Z".format(self.local_dir), + "docker.io/zanata/centos-repo-builder:{}".format( + self.dist_ver), + "-S", "/repo_host_dir/", + "-D", "/repo_host_dir/"] + + if version: + if version == 'auto': + version = GitHelper.detect_remote_repo_latest_version( + 'platform-', + 'https://github.com/zanata/zanata-platform.git') + logging.info( + "Update specfile %s to vesrsion %s ", + spec_file, version) + docker_run_cmd += ['-u', version] + + docker_run_cmd.append(spec_file) + logging.info("Run command: %s", " ".join(docker_run_cmd)) + subprocess.check_call(docker_run_cmd) + + +def main(argv=None): + # type (dict) -> None + """Run as command line program""" + if not argv: + argv = sys.argv[1:] + parser = ZanataArgParser(__file__) + parser.add_env('RPM_REPO_SSH_USER', dest='ssh_user') + parser.add_env('RPM_REPO_SSH_IDENTITY_FILE', dest='identity_file') + parser.add_methods_as_sub_commands( + RpmRepoHost, "pull|push|update_.*|all") + args = parser.parse_all(argv) + parser.run_sub_command(args) + + +if __name__ == '__main__': + if PROFILE: + import cProfile + import pstats + profile_filename = 'ZanataRpmRepo_profile.txt' + cProfile.run('main()', profile_filename) + statsfile = open("profile_stats.txt", "wb") + p = pstats.Stats(profile_filename, stream=statsfile) + stats = p.strip_dirs().sort_stats('cumulative') + stats.print_stats() + statsfile.close() + sys.exit(0) + main() diff --git a/jenkins/elrepo.jenkinsfile b/jenkins/elrepo.jenkinsfile new file mode 100644 index 0000000..1fedbe0 --- /dev/null +++ b/jenkins/elrepo.jenkinsfile @@ -0,0 +1,99 @@ +/** + * Jenkinsfile for Release RPM to Zanata_Team (a.k.a) dchen's repo + */ + +@Field +public static final String ORG_BASE = 'github.com/zanata' + +@Library('github.com/zanata/zanata-pipeline-library@v0.3.1') +import static org.zanata.jenkins.StackTraces.getStackTrace + +import groovy.transform.Field + +timestamps { + ansiColor('xterm') { + // We need a node with release label + node('release') { + currentBuild.displayName = currentBuild.displayName + " {${env.NODE_NAME}}" + + // To override the following variables, configure the pipeline job configuration in Jenkins, + // enable "Prepare an environment for the run", then fill in KEY=VALUE in "Properties Content" + String REPO_NAME = (env.REPO_NAME) ?: 'zanata-platform' + String ZANATA_SCRIPTS_BRANCH = (env.ZANATA_SCRIPTS_BRANCH) ?: 'master' + String LOCAL_HOME = sh( returnStdout: true, + script: "echo \$HOME").trim() + String WORK_ROOT = (env.WORK_ROOT) ?: "${LOCAL_HOME}/zanata-work-root" + String WORK_DIR = "${WORK_ROOT}/dnf/zanata" + String SPEC_FILE = "zanata-cli-bin.spec" + String VERBOSE = "DEBUG" + + def envArray = new ArrayList() + + def projectProperties = [ + [$class: 'BuildDiscarderProperty', + strategy: [$class: 'LogRotator', + daysToKeepStr: '', // keep records no more than X days + numToKeepStr: '10', // keep records for at most X builds + artifactDaysToKeepStr: '', // keep artifacts no more than X days + artifactNumToKeepStr: '', // keep artifacts for at most X builds + ] + ], + [$class: 'ParametersDefinitionProperty', + parameterDefinitions: [ + [$class: 'StringParameterDefinition', + defaultValue: 'auto', + description: 'Version to release like "4.7.0", or "auto" to release the latest', + name: 'RELEASE_VERSION' + ], + [$class: 'BooleanParameterDefinition', + defaultValue: false, + description: 'Push to dnf/yum repo', + name: 'PUSH_MODE' + ], + ] + ], + ] + + properties(projectProperties) + + if (params.PUSH_MODE == false ){ + currentBuild.displayName = currentBuild.displayName + " [no push]" + } + + stage('Checkout') { + // This checkout zanata-scripts + checkout scm + + envArray.addAll([ + "WORK_ROOT=${WORK_ROOT}", + ]) + } + + withEnv(envArray) { + sshagent (credentials: ['dchen.fedorapeople']) { + withCredentials( + [sshUserPrivateKey(credentialsId: 'dchen.fedorapeople', + keyFileVariable: 'RPM_REPO_SSH_IDENTITY_FILE', + passphraseVariable: 'RPM_REPO_PASS', + usernameVariable: 'RPM_REPO_SSH_USER')]) { + stage("Pull") { + sh "$WORKSPACE/ZanataRpmRepo.py pull -v DEBUG" + } + dir(WORK_DIR) { + stage("UpdateRepos") { + sh "$WORKSPACE/ZanataRpmRepo.py update_epel_repos -v ${VERBOSE} ${SPEC_FILE} ${params.RELEASE_VERSION}" + } + + if (params.PUSH_MODE) { + stage("Push") { + sh "$WORKSPACE/ZanataRpmRepo.py push -v ${VERBOSE}" + } + } + } + } + } + } + } + } +} + diff --git a/pylintrc b/pylintrc index d553452..0b1c2d1 100644 --- a/pylintrc +++ b/pylintrc @@ -22,3 +22,7 @@ enable= # Number of spaces of indent required inside a hanging or continued line. indent-after-paren=8 +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=7 \ No newline at end of file