LEXIO-37227: Package Generators

ns_poet/__init__.py

@@ -1,3 +1,3 @@
 """Manage Poetry packages in a monorepo"""
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"

ns_poet/config.py

@@ -0,0 +1,87 @@
+"""Contains a configuration parser and project config singleton"""
+from configparser import ConfigParser
+import json
+from pathlib import Path
+from typing import Any, FrozenSet, List
+
+from .util import get_git_top_level_path
+
+
+class Config:
+    """ns-poet configuration parser
+
+    Configuration is based on a .ns-poet.cfg file.
+
+    The schema for the file in the git project root is::
+
+        [project]
+        ignore_dirs = []
+        ignore_targets = []
+        python_package_name_prefix = ""
+        top_dirs = ["."]
+
+    The schema for the file in an individual python package is::
+
+        [package]
+        default_python_runtime = "python3.6"|"python3.7"|...
+
+    Instantiating this class will load and initialize a config parser instance.
+    """
+
+    def __init__(self, config_dir_path: Path) -> None:
+        """Initialize a config object
+
+        Args:
+            config_dir_path: path to a folder containing .ns-poet.cfg file
+
+        """
+        self.config_dir_path = config_dir_path
+        self._poet_cfg_path = Path(config_dir_path).joinpath(".ns-poet.cfg")
+        self._config = ConfigParser()
+        self._config["project"] = {}
+        self._config["package"] = {}
+        if self._poet_cfg_path.is_file():
+            self._config.read(self._poet_cfg_path)
+
+    def set(self, *args: Any) -> None:
+        """Proxy method to set a value on the underlying config parser instance"""
+        self._config.set(*args)
+
+    # Project options
+
+    @property
+    def ignore_dirs(self) -> FrozenSet[Any]:
+        """Never look for or process files in these directories"""
+        return frozenset(
+            json.loads(self._config.get("project", "ignore_dirs", fallback="[]"))
+        )
+
+    @property
+    def ignore_targets(self) -> FrozenSet[Any]:
+        """Set of target package names to ignore when collecting build targets"""
+        return frozenset(
+            json.loads(self._config.get("project", "ignore_targets", fallback="[]"))
+        )
+
+    @property
+    def python_package_name_prefix(self) -> str:
+        """Prefix to use for names of packages generated by ns-poet"""
+        return self._config.get("project", "python_package_name_prefix", fallback="")
+
+    @property
+    def top_dirs(self) -> List[str]:
+        """Top-level directories to search for Python packages"""
+        return json.loads(self._config.get("project", "top_dirs", fallback="['.']"))
+
+    # Package options
+
+    @property
+    def default_python_runtime(self) -> str:
+        """Default python runtime"""
+        return self._config.get(
+            "package", "default_python_runtime", fallback="python3.6"
+        )
+
+
+# Create a singleton for the project configuration
+PROJECT_CONFIG = Config(get_git_top_level_path())

ns_poet/generators/__init__.py

@@ -0,0 +1 @@
+"""Exported objects"""

ns_poet/generators/base.py

@@ -0,0 +1,177 @@
+"""Contains the base class for a package generator"""
+import inspect
+import logging
+import os
+from pathlib import Path
+import shutil
+from typing import Dict, Optional
+
+from cookiecutter.main import cookiecutter
+
+from ..config import PROJECT_CONFIG
+
+logger = logging.getLogger(__name__)
+
+
+class PackageGenerator:
+    """Base class for package generators.
+
+    A package generator is responsible for creating a folder in the repo with standard
+    boilerplate based on the type of package.
+
+    """
+
+    #: Package type must be defined in the child class
+    PACKAGE_TYPE: str = None
+    #: The top level directory is where the package will be stored. e.g. apps. It must
+    #: be defined in the child class.
+    TOP_LEVEL_DIR: str = None
+    #: The folder containing the new package can have a prefix. This will depend on the
+    #: package type.
+    FOLDER_NAME_PREFIX = ""
+
+    def __init__(
+        self,
+        nickname: str,
+        title: str,
+        description: str,
+        top_level_dir: Optional[str] = None,
+        template: Optional[str] = None,
+    ) -> None:
+        """Initializer
+
+        Args:
+            nickname: Nickname for the package. This will be used to folder names and
+                other identifiers.
+            title: Title of the package
+            description: Short description of the package
+            top_level_dir: The top level directory is where the package will be stored.
+                e.g. apps. Providing this during instantiation and not as a class
+                attribute should only be done for dynamic generator classes.
+            template: Template directory path to provide if you don't want it be
+                inferred. Providing this during instantiation and not as a class
+                attribute should only be done for dynamic generator classes.
+
+        """
+        self.nickname = nickname
+        self.title = title
+        self.description = description
+        self._top_level_dir = top_level_dir or self.TOP_LEVEL_DIR
+        # The new package will be created in this output folder
+        self._output_dir = PROJECT_CONFIG.config_dir_path.joinpath(self._top_level_dir)
+        # The generator template should be defined in a folder in the same directory as
+        # the generator module. Alternatively, it can be provided explicitly.
+        self._template = template or os.path.dirname(inspect.getfile(self.__class__))
+
+    @property
+    def folder_name(self) -> str:
+        """Basename of the folder for the new package"""
+        return self.FOLDER_NAME_PREFIX + self.nickname
+
+    @property
+    def package_dir(self) -> str:
+        """Absolute path to the new package directory"""
+        return os.path.join(
+            os.environ["TALOS_ROOT"], self._top_level_dir, self.folder_name
+        )
+
+    @property
+    def package_name(self) -> str:
+        """Name of the package. This is usually the name used for importing."""
+        return self.folder_name
+
+    @property
+    def build_dir(self) -> str:
+        """Directory containing the package's src files"""
+        return os.path.join(self._top_level_dir, self.folder_name)
+
+    @property
+    def package_path(self) -> str:
+        """Path to the package source code"""
+        return os.path.join(self.package_dir, "src", self.package_name)
+
+    @property
+    def context(self) -> Dict:
+        """Template context variables.
+
+        Child classes can override this method and extend the dict.
+        """
+        return {
+            "build_dir": self.build_dir,
+            "description": self.description,
+            "folder_name": self.folder_name,
+            "package_name": self.package_name,
+            "package_type": self.PACKAGE_TYPE,
+            "title": self.title,
+        }
+
+    def generate(self) -> None:
+        """Generate a new package in the repo"""
+        try:
+            cookiecutter(
+                self._template,
+                no_input=True,
+                extra_context=self.context,
+                replay=False,
+                overwrite_if_exists=True,
+                output_dir=self._output_dir,
+            )
+            self._post_generate()
+        except Exception as e:
+            logger.exception(f"Failed to generate the new package: {e}")
+            shutil.rmtree(self.package_dir, ignore_errors=True)
+            raise
+
+    def _post_generate(self) -> None:
+        """Optional post-generation step.
+
+        This may be overridden by child classes.
+        """
+        pass
+
+    def print_extra_help(self) -> None:
+        """Print extra help info to the terminal
+
+        This may be overridden by child classes.
+        """
+        pass
+
+
+class PythonPackageGenerator(PackageGenerator):
+    """Generator for a generic Python package.
+
+    This class is meant to be extended.
+    """
+
+    PACKAGE_TYPE = "python"
+
+    @property
+    def folder_name(self) -> str:
+        """Basename of the folder for the new package"""
+        return super().folder_name.replace("-", "_")
+
+    @property
+    def package_name(self) -> str:
+        """Name of the package. This is usually the name used for importing."""
+        return f"{PROJECT_CONFIG.python_package_name_prefix}{self.folder_name}"
+
+    @property
+    def build_dir(self) -> str:
+        """Directory containing the package's BUILD file"""
+        return os.path.join(super().build_dir, "src")
+
+    def _post_generate(self) -> None:
+        """Cleanup the generated package"""
+        self._clean_init_files()
+
+    def _clean_init_files(self) -> None:
+        """For some reason a couple __init__.py files get generated. Remove them."""
+        try:
+            root = Path(self.package_dir)
+            files = list(root.rglob("*.pyc"))
+            files.append(root.joinpath("__init__.py"))
+            files.append(root.joinpath("src/__init__.py"))
+            for f in files:
+                f.unlink()
+        except Exception:
+            pass

ns_poet/generators/load.py

@@ -0,0 +1,72 @@
+"""Contains functions for loading generator plugins"""
+import importlib
+import logging
+import sys
+from typing import Dict
+
+from ..config import PROJECT_CONFIG  # noqa: I100
+from .base import PackageGenerator  # noqa: I100
+
+logger = logging.getLogger(__name__)
+
+
+def load_project_generators() -> Dict[str, PackageGenerator]:
+    """Load package generators defined in the project.
+
+    Returns:
+        dict of generator name (generator folder name) to package generator class
+
+    """
+    generators_path = PROJECT_CONFIG.config_dir_path.joinpath(".ns-poet/generators")
+    generators = {}
+    if generators_path.is_dir():
+        sys.path.append(str(generators_path))
+        for entry in generators_path.iterdir():
+            if entry.is_dir():
+                try:
+                    mod = importlib.import_module(entry.name)
+                    generators[entry.name] = mod.Generator  # type: ignore  # noqa
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to find a Generator class in {entry}: {str(e)}"
+                    )
+
+    return generators
+
+
+# Create a singleton of package generator classes
+PACKAGE_GENERATOR_MAP = load_project_generators()
+
+
+def create_package_generator(
+    package_type: str,
+    nickname: str,
+    title: str,
+    description: str,
+    top_level_dir: str = None,
+    template: str = None,
+) -> PackageGenerator:
+    """Factory function for creating a package generator object.
+
+    Args:
+        package_type: Package type that corresponds to a key in the package generator map
+        nickname: Nickname for the package. This will be used to folder names and
+            other identifiers.
+        title: Title of the package
+        description: Short description of the package
+        top_level_dir: The top level directory is where the package will be stored.
+            e.g. apps. Providing this during instantiation and not as a class
+            attribute should only be done for dynamic generator classes.
+        template: Template directory path to provide if you don't want it be
+            inferred. Providing this during instantiation and not as a class
+            attribute should only be done for dynamic generator classes.
+
+    Returns:
+        new package generator object
+
+    """
+    PackageGeneratorClass: PackageGenerator = PACKAGE_GENERATOR_MAP[package_type]
+    generator: PackageGenerator = PackageGeneratorClass(  # type: ignore  # noqa
+        nickname, title, description, top_level_dir=top_level_dir, template=template
+    )
+    return generator

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ns-poet"
-version = "0.3.0"
+version = "0.4.0"
 description = "Autogenerate Poetry package manifests in a monorepo"
 authors = ["Jonathan Drake <jdrake@narrativescience.com>"]
 license = "BSD-3-Clause"