pavement.py

r"""
This paver file is intended to help with the release process as much as
possible. It relies on virtualenv to generate 'bootstrap' environments as
independent from the user system as possible (e.g. to make sure the sphinx doc
is built against the built numpy, not an installed one).

Building changelog + notes
==========================

Assumes you have git and the binaries/tarballs in installers/::

    paver write_release
    paver write_note

This automatically put the checksum into README.rst, and writes the Changelog.

TODO
====
    - the script is messy, lots of global variables
    - make it more easily customizable (through command line args)
    - missing targets: install & test, sdist test, debian packaging
    - fix bdist_mpkg: we build the same source twice -> how to make sure we use
      the same underlying python for egg install in venv and for bdist_mpkg
"""
from __future__ import division, print_function

import os
import sys
import shutil
import subprocess
import re
import hashlib

# The paver package needs to be installed to run tasks
import paver
from paver.easy import Bunch, options, task, sh


#-----------------------------------
# Things to be changed for a release
#-----------------------------------

# Path to the release notes
RELEASE_NOTES = 'doc/source/release/1.18.0-notes.rst'


#-------------------------------------------------------
# Hardcoded build/install dirs, virtualenv options, etc.
#-------------------------------------------------------

# Where to put the release installers
options(installers=Bunch(releasedir="release",
                         installersdir=os.path.join("release", "installers")),)


#-----------------------------
# Generate the release version
#-----------------------------

sys.path.insert(0, os.path.dirname(__file__))
try:
    setup_py = __import__("setup")
    FULLVERSION = setup_py.VERSION
    # This is duplicated from setup.py
    if os.path.exists('.git'):
        GIT_REVISION = setup_py.git_version()
    elif os.path.exists('numpy/version.py'):
        # must be a source distribution, use existing version file
        from numpy.version import git_revision as GIT_REVISION
    else:
        GIT_REVISION = "Unknown"

    if not setup_py.ISRELEASED:
        FULLVERSION += '.dev0+' + GIT_REVISION[:7]
finally:
    sys.path.pop(0)


#--------------------------
# Source distribution stuff
#--------------------------
def tarball_name(ftype='gztar'):
    """Generate source distribution name

    Parameters
    ----------
    ftype : {'zip', 'gztar'}
        Type of archive, default is 'gztar'.

    """
    root = 'numpy-%s' % FULLVERSION
    if ftype == 'gztar':
        return root + '.tar.gz'
    elif ftype == 'zip':
        return root + '.zip'
    raise ValueError("Unknown type %s" % type)

@task
def sdist(options):
    """Make source distributions.

    Parameters
    ----------
    options :
        Set by ``task`` decorator.

    """
    # First clean the repo and update submodules (for up-to-date doc html theme
    # and Sphinx extensions)
    sh('git clean -xdf')
    sh('git submodule init')
    sh('git submodule update')

    # To be sure to bypass paver when building sdist... paver + numpy.distutils
    # do not play well together.
    # Cython is run over all Cython files in setup.py, so generated C files
    # will be included.
    sh('python3 setup.py sdist --formats=gztar,zip')

    # Copy the superpack into installers dir
    idirs = options.installers.installersdir
    if not os.path.exists(idirs):
        os.makedirs(idirs)

    for ftype in ['gztar', 'zip']:
        source = os.path.join('dist', tarball_name(ftype))
        target = os.path.join(idirs, tarball_name(ftype))
        shutil.copy(source, target)


#-------------
# README stuff
#-------------

def _compute_hash(idirs, hashfunc):
    """Hash files using given hashfunc.

    Parameters
    ----------
    idirs : directory path
        Directory containing files to be hashed.
    hashfunc : hash function
        Function to be used to hash the files.

    """
    released = paver.path.path(idirs).listdir()
    checksums = []
    for fpath in sorted(released):
        with open(fpath, 'rb') as fin:
            fhash = hashfunc(fin.read())
            checksums.append(
                '%s  %s' % (fhash.hexdigest(), os.path.basename(fpath)))
    return checksums


def compute_md5(idirs):
    """Compute md5 hash of files in idirs.

    Parameters
    ----------
    idirs : directory path
        Directory containing files to be hashed.

    """
    return _compute_hash(idirs, hashlib.md5)


def compute_sha256(idirs):
    """Compute sha256 hash of files in idirs.

    Parameters
    ----------
    idirs : directory path
        Directory containing files to be hashed.

    """
    # better checksum so gpg signed README.rst containing the sums can be used
    # to verify the binaries instead of signing all binaries
    return _compute_hash(idirs, hashlib.sha256)


def write_release_task(options, filename='README'):
    """Append hashes of release files to release notes.

    This appends file hashes to the release notes ane creates
    four README files of the result in various formats:

    - README.rst
    - README.rst.gpg
    - README.md
    - README.md.gpg

    The md file are created using `pandoc` so that the links are
    properly updated. The gpg files are kept separate, so that
    the unsigned files may be edited before signing if needed.

    Parameters
    ----------
    options :
        Set by ``task`` decorator.
    filename : string
        Filename of the modified notes. The file is written
        in the release directory.

    """
    idirs = options.installers.installersdir
    notes = paver.path.path(RELEASE_NOTES)
    rst_readme = paver.path.path(filename + '.rst')
    md_readme = paver.path.path(filename + '.md')

    # append hashes
    with open(rst_readme, 'w') as freadme:
        with open(notes) as fnotes:
            freadme.write(fnotes.read())

        freadme.writelines("""
Checksums
=========

MD5
---
::

""")
        freadme.writelines([f'    {c}\n' for c in compute_md5(idirs)])
        freadme.writelines("""
SHA256
------
::

""")
        freadme.writelines([f'    {c}\n' for c in compute_sha256(idirs)])

    # generate md file using pandoc before signing
    sh(f"pandoc -s -o {md_readme} {rst_readme}")

    # Sign files
    if hasattr(options, 'gpg_key'):
        cmd = f'gpg --clearsign --armor --default_key {options.gpg_key}'
    else:
        cmd = 'gpg --clearsign --armor'

    sh(cmd + f' --output {rst_readme}.gpg {rst_readme}')
    sh(cmd + f' --output {md_readme}.gpg {md_readme}')


@task
def write_release(options):
    """Write the README files.

    Two README files are generated from the release notes, one in ``rst``
    markup for the general release, the other in ``md`` markup for the github
    release notes.

    Parameters
    ----------
    options :
        Set by ``task`` decorator.

    """
    rdir = options.installers.releasedir
    write_release_task(options, os.path.join(rdir, 'README'))