Packaging tools (#2618)

* Rename testutils to tools

* Move packaging requirement

* Alpha of regenerate packaging, hardcoded for scheduler

* Configuration generation

* Compute packaging configuration

* Scheduler packaging configuration

azure-mgmt-compute/README.rst

@@ -6,7 +6,7 @@ This is the Microsoft Azure Compute Management Client Library.
 Azure Resource Manager (ARM) is the next generation of management APIs that
 replace the old Azure Service Management (ASM).
 
-This package has been tested with Python 2.7, 3.3, 3.4, 3.5 and 3.6.
+This package has been tested with Python 2.7, 3.4, 3.5 and 3.6.
 
 For the older Azure Service Management (ASM) libraries, see
 `azure-servicemanagement-legacy <https://pypi.python.org/pypi/azure-servicemanagement-legacy>`__ library.
@@ -36,7 +36,7 @@ If you see azure==0.11.0 (or any version below 1.0), uninstall it first:
 Usage
 =====
 
-For code examples, see `Compute and Network Resource Management 
+For code examples, see `Compute Management
 <https://docs.microsoft.com/python/api/overview/azure/virtualmachines>`__
 on docs.microsoft.com.
 

azure-mgmt-compute/sdk_packaging.toml

@@ -0,0 +1,5 @@
+[packaging]
+package_name = "azure-mgmt-compute"
+package_pprint_name = "Compute Management"
+package_doc_id = "virtualmachines"
+is_stable = true

azure-mgmt-compute/setup.py

@@ -77,7 +77,7 @@
     zip_safe=False,
     packages=find_packages(exclude=["tests"]),
     install_requires=[
-        'msrestazure>=0.4.20,<2.0.0',
+        'msrestazure>=0.4.27,<2.0.0',
         'azure-common~=1.1',
     ],
     cmdclass=cmdclass

azure-mgmt-scheduler/sdk_packaging.toml

@@ -0,0 +1,6 @@
+[packaging]
+package_name = "azure-mgmt-scheduler"
+package_pprint_name = "Scheduler Management"
+package_doc_id = "scheduler"
+is_stable = true
+

scripts/packaging_requirements.txt → azure-sdk-tools/packaging_requirements.txt

@@ -1,3 +1,4 @@
-cookiecutter
 packaging
 wheel
+Jinja2
+pytoml

azure-sdk-tools/packaging_tools/__init__.py

@@ -0,0 +1,62 @@
+import logging
+from pathlib import Path
+from typing import Dict, Any
+
+from jinja2 import Template, PackageLoader, Environment
+from .conf import read_conf, build_default_conf, CONF_NAME
+
+_LOGGER = logging.getLogger(__name__)
+
+_CWD = Path(__file__).resolve().parent
+_TEMPLATE_PATH = _CWD / "template"
+
+def build_config(config : Dict[str, Any]) -> Dict[str, str]:
+    """Will build the actual config for Jinja2, based on SDK config.
+    """
+    result = config.copy()
+    # Manage the classifier stable/beta
+    is_stable = result.pop("is_stable", False)
+    if is_stable:
+        result["classifier"] = "Development Status :: 5 - Production/Stable"
+    else:
+        result["classifier"] = "Development Status :: 4 - Beta"
+    # Manage the nspkg
+    package_name = result["package_name"]
+    result["package_nspkg"] = package_name[:package_name.rindex('-')]+"-nspkg"
+
+    # Return result
+    return result
+
+def build_packaging(package_name: str, output_folder: str, build_conf: bool = False) -> None:
+    _LOGGER.info("Building template %s", package_name)
+    package_folder = Path(output_folder) / Path(package_name)
+
+    if build_conf:
+        build_default_conf(package_folder, package_name)
+
+    conf = read_conf(package_folder)
+    if not conf:
+        raise ValueError("Create a {} file before calling this script".format(package_folder / CONF_NAME))
+
+    env = Environment(
+        loader=PackageLoader('packaging_tools', 'templates'),
+        keep_trailing_newline=True
+    )
+    conf = build_config(conf)
+
+    for template_name in env.list_templates():
+        future_filepath = Path(output_folder) / package_name / template_name
+
+        # Might decide to make it more generic one day
+        if template_name == "HISTORY.rst" and future_filepath.exists():
+            _LOGGER.info("Skipping HISTORY.txt template, since a previous one was found")
+            # Never overwirte the ChangeLog
+            continue
+
+        template = env.get_template(template_name)
+        result = template.render(**conf)
+
+        with open(Path(output_folder) / package_name / template_name, "w") as fd:
+            fd.write(result)
+
+    _LOGGER.info("Template done %s", package_name)

azure-sdk-tools/packaging_tools/__main__.py

@@ -0,0 +1,41 @@
+import argparse
+import logging
+import sys
+
+from . import build_packaging
+
+_LOGGER = logging.getLogger(__name__)
+
+_epilog="""This script will automatically build the TOML configuration file with default value if it doesn't exist.
+"""
+
+parser = argparse.ArgumentParser(
+    description='Packaging tools for Azure SDK for Python',
+    formatter_class=argparse.RawTextHelpFormatter,
+    epilog=_epilog
+)
+parser.add_argument('--output', '-o',
+                    dest='output', default='.',
+                    help='Output dir, should be SDK repo folder. [default: %(default)s]')
+parser.add_argument("--debug",
+                    dest="debug", action="store_true",
+                    help="Verbosity in DEBUG mode")
+parser.add_argument("--build-conf",
+                    dest="build_conf", action="store_true",
+                    help="Build a default TOML file, with package name, fake pretty name, as beta package and no doc page. Do nothing if the file exists, remove manually the file if needed.")
+parser.add_argument('package_name', help='The package name.')
+
+args = parser.parse_args()
+
+main_logger = logging.getLogger()
+logging.basicConfig()
+main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
+
+try:
+    build_packaging(args.package_name, args.output, build_conf=args.build_conf)
+except Exception as err:
+    if args.debug:
+        _LOGGER.exception(err)
+    else:
+        _LOGGER.critical(err)
+    sys.exit(1)

azure-sdk-tools/packaging_tools/conf.py

@@ -0,0 +1,39 @@
+import logging
+from pathlib import Path
+from typing import Dict, Any
+
+import pytoml as toml
+
+_LOGGER = logging.getLogger(__name__)
+
+CONF_NAME = "sdk_packaging.toml"
+_SECTION = "packaging"
+
+# Default conf
+_CONFIG = {
+    "package_name": "packagename",
+    "package_pprint_name": "MyService Management",
+    "package_doc_id": "",
+    "is_stable": False
+}
+
+def read_conf(folder: Path) -> Dict[str, Any]:
+    conf_path = folder / CONF_NAME
+    if not conf_path.exists():
+        return {}
+
+    with open(conf_path, "rb") as fd:
+        return toml.load(fd)[_SECTION]
+
+def build_default_conf(folder: Path, package_name: str) -> None:
+    conf_path = folder / CONF_NAME
+    if conf_path.exists():
+        _LOGGER.info("Skipping default conf since the file exists")
+        return
+
+    _LOGGER.info("Build default conf for %s", package_name)
+    conf = {_SECTION: _CONFIG.copy()}
+    conf[_SECTION]["package_name"] = package_name
+
+    with open(conf_path, "w") as fd:
+        toml.dump(conf, fd)

azure-sdk-tools/packaging_tools/templates/HISTORY.rst

@@ -0,0 +1,9 @@
+.. :changelog:
+
+Release History
+===============
+
+0.1.0 (1970-01-01)
+++++++++++++++++++
+
+* Initial Release

azure-sdk-tools/packaging_tools/templates/MANIFEST.in

@@ -0,0 +1,2 @@
+include *.rst
+include azure_bdist_wheel.py

azure-sdk-tools/packaging_tools/templates/README.rst

@@ -0,0 +1,49 @@
+Microsoft Azure SDK for Python
+==============================
+
+This is the Microsoft Azure {{package_pprint_name}} Client Library.
+
+Azure Resource Manager (ARM) is the next generation of management APIs that
+replace the old Azure Service Management (ASM).
+
+This package has been tested with Python 2.7, 3.4, 3.5 and 3.6.
+
+For the older Azure Service Management (ASM) libraries, see
+`azure-servicemanagement-legacy <https://pypi.python.org/pypi/azure-servicemanagement-legacy>`__ library.
+
+For a more complete set of Azure libraries, see the `azure <https://pypi.python.org/pypi/azure>`__ bundle package.
+
+
+Compatibility
+=============
+
+**IMPORTANT**: If you have an earlier version of the azure package
+(version < 1.0), you should uninstall it before installing this package.
+
+You can check the version using pip:
+
+.. code:: shell
+
+    pip freeze
+
+If you see azure==0.11.0 (or any version below 1.0), uninstall it first:
+
+.. code:: shell
+
+    pip uninstall azure
+
+
+Usage
+=====
+
+For code examples, see `{{package_pprint_name}}
+<https://docs.microsoft.com/python/api/overview/azure/{{package_doc_id}}>`__
+on docs.microsoft.com.
+
+
+Provide Feedback
+================
+
+If you encounter any bugs or have suggestions, please file an issue in the
+`Issues <https://github.com/Azure/azure-sdk-for-python/issues>`__
+section of the project.

azure-sdk-tools/packaging_tools/templates/azure_bdist_wheel.py

@@ -0,0 +1,54 @@
+#-------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+#--------------------------------------------------------------------------
+
+from distutils import log as logger
+import os.path
+
+from wheel.bdist_wheel import bdist_wheel
+class azure_bdist_wheel(bdist_wheel):
+    """The purpose of this class is to build wheel a little differently than the sdist,
+    without requiring to build the wheel from the sdist (i.e. you can build the wheel
+    directly from source).
+    """
+
+    description = "Create an Azure wheel distribution"
+
+    user_options = bdist_wheel.user_options + \
+        [('azure-namespace-package=', None,
+            "Name of the deepest nspkg used")]
+
+    def initialize_options(self):
+        bdist_wheel.initialize_options(self)
+        self.azure_namespace_package = None
+
+    def finalize_options(self):
+        bdist_wheel.finalize_options(self)
+        if self.azure_namespace_package and not self.azure_namespace_package.endswith("-nspkg"):
+            raise ValueError("azure_namespace_package must finish by -nspkg")
+
+    def run(self):
+        if not self.distribution.install_requires:
+            self.distribution.install_requires = []
+        self.distribution.install_requires.append(
+            "{}>=2.0.0".format(self.azure_namespace_package))
+        bdist_wheel.run(self)
+
+    def write_record(self, bdist_dir, distinfo_dir):
+        if self.azure_namespace_package:
+            # Split and remove last part, assuming it's "nspkg"
+            subparts = self.azure_namespace_package.split('-')[0:-1]
+        folder_with_init = [os.path.join(*subparts[0:i+1]) for i in range(len(subparts))]
+        for azure_sub_package in folder_with_init:
+            init_file = os.path.join(bdist_dir, azure_sub_package, '__init__.py')
+            if os.path.isfile(init_file):
+                logger.info("manually remove {} while building the wheel".format(init_file))
+                os.remove(init_file)
+            else:
+                raise ValueError("Unable to find {}. Are you sure of your namespace package?".format(init_file))
+        bdist_wheel.write_record(self, bdist_dir, distinfo_dir)
+cmdclass = {
+    'bdist_wheel': azure_bdist_wheel,
+}

azure-sdk-tools/packaging_tools/templates/setup.cfg

@@ -0,0 +1,3 @@
+[bdist_wheel]
+universal=1
+azure-namespace-package={{package_nspkg}}