plugin-template

#!/usr/bin/env python3

import argparse
import os
import pprint
import shutil
import stat
import sys
import tempfile
import textwrap
import utils
import yaml
from pathlib import Path

from jinja2 import Environment, FileSystemLoader


DEFAULT_SETTINGS = {
    'additional_repos': [],
    'black': True,
    'check_commit_message': True,
    'check_gettext': True,
    'check_manifest': True,
    'check_stray_pulpcore_imports': True,
    'coverage': False,
    'deploy_client_to_pypi': True,
    'deploy_client_to_rubygems': True,
    'deploy_daily_client_to_pypi': True,
    'deploy_daily_client_to_rubygems': True,
    'deploy_to_pypi': True,
    'docker_fixtures': False,
    'docs_test': True,
    'plugin_app_label': None,
    'plugin_camel': None,
    'plugin_camel_short': None,
    'plugin_caps': None,
    'plugin_caps_short': None,
    'plugin_dash': None,
    'plugin_dash_short': None,
    'plugin_default_branch': 'master',
    'plugin_name': None,
    'plugin_snake': None,
    'publish_docs_to_pulpprojectdotorg': False,
    'pulp_settings': None,
    'pulpcore_branch': 'master',
    'pulpcore_pip_version_specifier': None,
    'pulpprojectdotorg_key_id': None,
    'pydocstyle': True,
    'pypi_username': None,
    'redmine_project': None,
    'test_bindings': True,
    'test_cli': False,
    'test_performance': False,
    'test_released_plugin_with_next_pulpcore_release': False,
    'test_s3': False,
    'update_redmine': False,
}


DEPRECATED_FILES = [
    ".ci/scripts/cherrypick.sh",
    ".ci/scripts/cherrypick.py",
    ".ci/scripts/release_requirements.txt",
    ".github/workflows/fips.yml",
    ".travis",
    ".travis.yml",
]


def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
                                     description='Create or update a plugin using the current '
                                                 'template.')
    parser.add_argument('plugin_name', type=str,
                        help=textwrap.dedent('''\
                            Create or update this plugin. The name should start with pulp- or pulp_.

                        '''))
    parser.add_argument('--plugin-app-label', type=str,
                        help=textwrap.dedent('''\
                            the Django app label for the plugin - usually the part after pulp_ or
                            pulp-.

                        '''))
    parser.add_argument('--generate-config', action='store_true',
                        help=textwrap.dedent('''\
                            Create or update a plugin template config for a plugin and exit.

                        '''))
    parser.add_argument('--bootstrap', action='store_true',
                        help=textwrap.dedent('''\
                            Create a new plugin and template boilerplate code.

                        '''))
    parser.add_argument('--test', action='store_true',
                        help=textwrap.dedent('''\
                            Generate or update functional and unit tests.

                        '''))
    parser.add_argument('--github', action='store_true',
                        help=textwrap.dedent('''\
                            Generate or update github CI/CD configuration files.

                        '''))
    parser.add_argument('--docs', action='store_true',
                        help=textwrap.dedent('''\
                            Generate or update plugin documentation.

                        '''))
    parser.add_argument('--all', action='store_true',
                        help=textwrap.dedent('''\
                            Create a new plugin and template all non-excluded files.

                        '''))
    parser.add_argument('--verbose', action='store_true',
                        help=textwrap.dedent('''\
                            Include more output.

                        '''))
    args = parser.parse_args()



    here = os.path.dirname(os.path.abspath(__file__))
    plugin_root_dir = os.path.join(os.path.dirname(here), args.plugin_name)
    plugin_config_path = os.path.join(plugin_root_dir, 'template_config.yml')
    write_new_config = False

    try:
        with open(plugin_config_path) as config_file:
            try:
                config_in_file = yaml.safe_load(config_file)
                if config_in_file:
                    # TODO: validate config
                    config = config_in_file
                    # Add any missing value from the list of defaults
                    for key, value in DEFAULT_SETTINGS.items():
                        if key not in config:
                            config[key] = value
                            write_new_config = True
                    print("\nLoaded plugin template config from "
                          "{path}/template_config.yml.\n".format(path=plugin_root_dir))
                    if args.verbose:
                        print("\nUsing the following config:\n")
                        pp = pprint.PrettyPrinter(indent=4)
                        pp.pprint(config)
                        print("\n")
            except yaml.YAMLError as exc:
                print(exc)
                exit()
    except FileNotFoundError:
        print("\nCould not find a plugin template config at {path}/template_config.yml.\n".format(
            path=plugin_root_dir))
        if args.all or args.generate_config:
            if not args.plugin_app_label:
                print("\n--plugin-app-label needs to be specified when generating a template "
                      "config. See ./plugin-template -h for usage.\n")
                return 2
            else:
                config = generate_config(args.plugin_name, args.plugin_app_label)
        else:
            return 2

    # Config key is used by the template_config.yml.j2 template to dump
    # the config. (note: uses .copy() to avoid a self reference)
    config['config'] = config.copy()

    if args.github or args.all:
        if not config['pypi_username'] and (config['deploy_client_to_pypi'] or \
                config['deploy_to_pypi']):
            print("If PyPI scripts are included, PyPI username is required.")
            return 2

    for section in ['generate_config', 'bootstrap', 'github', 'docs', 'test']:
        if getattr(args, section) or args.all:
            write_template_section(config, section, plugin_root_dir, verbose=args.verbose)

    if write_new_config and not (args.generate_config or args.all):
        write_template_section(config, 'generate_config', plugin_root_dir, verbose=args.verbose)
        file_path = os.path.join(plugin_root_dir, 'template_config.yml')
        print("\nAn updated plugin template config written to {path}.\n".format(path=file_path))

    if plugin_root_dir:
        print("\nDeprecation check:")
        check_for_deprecated_files(plugin_root_dir)


def to_nice_yaml(data):
    """Implement a filter for Jinja 2 templates to render human readable YAML."""
    return yaml.dump(data, indent=2, allow_unicode=True, default_flow_style=False)


def write_template_section(config, name, plugin_root_dir, verbose=False):
    """
    Template or copy all files for the section.
    """
    section_template_dir = "templates/{name}".format(name=name)
    env = Environment(
        loader=FileSystemLoader([section_template_dir, 'templates'])
    )

    files_templated = 0
    files_copied = 0
    for relative_path in generate_relative_path_set(section_template_dir):
        destination_relative_path = relative_path.replace('plugin_name', config['plugin_snake'])
        necessary_dir_structure = os.path.dirname(os.path.join(plugin_root_dir, destination_relative_path))

        if not os.path.exists(necessary_dir_structure):
            os.makedirs(necessary_dir_structure)

        if relative_path.endswith('.j2'):
            env.filters['to_yaml'] = to_nice_yaml
            template = env.get_template(relative_path)
            destination = destination_relative_path[:-len('.j2')]
            write_template_to_file(template, plugin_root_dir, destination, {"section": name, **config})
            files_templated += 1
            if verbose:
                print(f"Templated file: {relative_path}")
        else:
            if destination_relative_path.endswith('.copy'):
                destination_relative_path = destination_relative_path[:-len(".copy")]
            shutil.copyfile(
                os.path.join(section_template_dir, relative_path),
                os.path.join(plugin_root_dir, destination_relative_path)
            )
            files_copied += 1
            if verbose:
                print(f"Copied file: {relative_path}")

    print(f"Section: {name} \n    templated: {files_templated}\n    copied: {files_copied}")
    return 0


def generate_relative_path_set(root_dir):
    """
    Create a set of relative paths within the specified directory.
    """
    applicable_paths = set()
    for root, dirs, files in os.walk(root_dir, topdown=False):
        for file_name in files:
            template_abs_path = os.path.join(root, file_name)
            template_relative_path = os.path.relpath(template_abs_path,
                                                     root_dir)
            applicable_paths.add(template_relative_path)
    return applicable_paths


def write_template_to_file(template, plugin_root_dir, relative_path, config):
    """
    Render template with values from the config and write it to the target plugin directory.
    """

    with tempfile.NamedTemporaryFile(mode='w', dir=plugin_root_dir, delete=False) as fd_out:
        tempfile_path = fd_out.name
        fd_out.write(template.render(**config))
        fd_out.write('\n')

        destination_path = os.path.normpath(os.path.join(plugin_root_dir, relative_path))
        os.rename(tempfile_path, destination_path)

        if destination_path.endswith('.sh'):
            mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | \
                stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
        else:
            mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH
        os.chmod(destination_path, mode)


def generate_config(plugin_name, plugin_app_label):
    """
    Generates a default config for a plugin

    Args:
        plugin_name (str): the full name of the plugin repository name (e.g. pulp_file)
        plugin_app_label (str): the Django app label for the plugin - usually the part after
            pulp_ or pulp-

    Returns:
        config (dict): a dictionary containing a default template configuration for the plugin
    """
    config = {}
    config['plugin_name'] = plugin_name
    config['plugin_app_label'] = plugin_app_label
    config['plugin_snake'] = utils.to_snake(plugin_name)
    config['plugin_caps'] = utils.to_caps(plugin_name)
    config['plugin_caps_short'] = utils.to_caps(plugin_app_label)
    config['plugin_camel'] = utils.to_camel(plugin_name)
    config['plugin_camel_short'] = utils.to_camel(plugin_app_label)
    config['plugin_dash'] = utils.to_dash(plugin_name)
    config['plugin_dash_short'] = utils.to_dash(plugin_app_label)
    # Add any missing value from the list of defaults
    for key, value in DEFAULT_SETTINGS.items():
        if key not in config:
            config[key] = value
    return config


def check_for_deprecated_files(plugin_root_dir):
    """Check for files that have been deprecated (ie moved or removed)."""
    files_found = False
    for fp in DEPRECATED_FILES:
        path = Path(plugin_root_dir).joinpath(fp)
        if path.exists():
            print(f"Found deprecated file: '{path}'. It is recommended you delete this file.")
            files_found = True

    if not files_found:
        print("No deprecated files found.")


if __name__ == '__main__':
    sys.exit(main())