feat: Upload distribution files to GitHub Releases (#177)

* refactor(github): create upload_asset function

Create a function to call the asset upload API. This will soon be used 
to upload assets specified by the user.

* refactor(github): infer Content-Type from file extension

Infer the Content-Type header based on the file extension instead of setting it manually.

* refactor(pypi): move building of dists to cli.py

Refactor to have the building/removal of distributions in cli.py instead 
of within the upload_to_pypi function. This makes way for uploading to 
other locations, such as GitHub Releases, too.

* feat(github): upload dists to release

Upload Python wheels to the GitHub release. Configured with the option 
upload_to_release, on by default if using GitHub.

* docs: document upload_to_release config option

* test(github): add tests for Github.upload_dists

* fix(github): fix upload of .whl files

Fix uploading of .whl files due to a missing MIME type (defined custom type as application/x-wheel+zip). Additionally, continue with other uploads even if one fails.

* refactor(cli): additional output during publish

Add some additional output during the publish command.

* refactor(github): move api calls to separate methods

Move each type of GitHub API request into its own method to improve readability.

Re-implementation of #172

* fix: post changelog after PyPI upload

Post the changelog in-between uploading to PyPI and uploading to GitHub Releases. This is so that if the PyPI upload fails, GitHub users will not be notified. GitHub uploads still need to be processed after creating the changelog as the release notes must be published to upload assets to them.

docs/configuration.rst

@@ -39,6 +39,10 @@ Moreover, those configuration values can be overloaded with the ``-D`` option, l
     If set to false the pypi uploading will be disabled. This can be useful to create
     tag releases for non-pypi projects.
 
+``upload_to_release``
+    If set to false, do not upload distributions to GitHub releases. If you are not using
+    GitHub, this will be skipped regardless.
+
 ``commit_message``
     Long description to append to the version number. This can be useful to skip
     pipelines in your CI tool

semantic_release/cli.py

@@ -9,10 +9,12 @@
 from semantic_release import ci_checks
 from semantic_release.errors import GitError, ImproperConfigurationError
 
+from .dist import build_dists, remove_dists
 from .history import (evaluate_version_bump, get_current_version, get_new_version,
                       get_previous_version, set_new_version)
 from .history.logs import generate_changelog, markdown_changelog
-from .hvcs import check_build_status, check_token, get_domain, get_token, post_changelog
+from .hvcs import (check_build_status, check_token, get_domain, get_token, post_changelog,
+                   upload_to_release)
 from .pypi import upload_to_pypi
 from .settings import config, overload_configuration
 from .vcs_helpers import (checkout, commit_new_version, get_current_head_hash,
@@ -169,6 +171,7 @@ def publish(**kwargs):
     checkout(branch)
 
     if version(**kwargs):
+        click.echo('Pushing new version')
         push_new_version(
             auth_token=get_token(),
             owner=owner,
@@ -177,14 +180,26 @@ def publish(**kwargs):
             domain=get_domain(),
         )
 
-        if config.getboolean('semantic_release', 'upload_to_pypi'):
+        # Get config options for uploads
+        dist_path = config.get('semantic_release', 'dist_path')
+        remove_dist = config.getboolean('semantic_release', 'remove_dist')
+        upload_pypi = config.getboolean('semantic_release', 'upload_to_pypi')
+        upload_release = config.getboolean('semantic_release', 'upload_to_release')
+
+        if upload_pypi or upload_release:
+            click.echo('Building distributions')
+            if remove_dist:
+                # Remove old distributions before building
+                remove_dists(dist_path)
+            build_dists()
+        if upload_pypi:
+            click.echo('Uploading to PyPI')
             upload_to_pypi(
+                path=dist_path,
                 username=os.environ.get('PYPI_USERNAME'),
                 password=os.environ.get('PYPI_PASSWORD'),
                 # We are retrying, so we don't want errors for files that are already on PyPI.
-                skip_existing=retry,
-                remove_dist=config.getboolean('semantic_release', 'remove_dist'),
-                path=config.get('semantic_release', 'dist_path'),
+                skip_existing=retry
             )
 
         if check_token():
@@ -199,11 +214,18 @@ def publish(**kwargs):
                 )
             except GitError:
                 click.echo(click.style('Posting changelog failed.', 'red'), err=True)
-
         else:
             click.echo(
                 click.style('Missing token: cannot post changelog', 'red'), err=True)
 
+        # Upload to GitHub Releases
+        if upload_release and check_token():
+            click.echo('Uploading to HVCS release')
+            upload_to_release(owner, name, new_version, dist_path)
+        # Remove distribution files as they are no longer needed
+        if remove_dist:
+            remove_dists(dist_path)
+
         click.echo(click.style('New release published', 'green'))
     else:
         click.echo('Version failed, no release will be published.', err=True)

semantic_release/defaults.cfg

@@ -6,6 +6,7 @@ check_build_status=false
 hvcs=github
 commit_parser=semantic_release.history.angular_parser
 upload_to_pypi=true
+upload_to_release=true
 version_source=commit
 commit_message=Automatically generated by python-semantic-release
 dist_path=dist

semantic_release/dist.py

@@ -0,0 +1,11 @@
+"""Build and manage distributions
+"""
+from invoke import run
+
+
+def build_dists():
+    run('python setup.py sdist bdist_wheel')
+
+
+def remove_dists(path: str):
+    run(f'rm -rf {path}')

semantic_release/hvcs.py

@@ -1,5 +1,6 @@
 """HVCS
 """
+import mimetypes
 import os
 from typing import Optional
 
@@ -14,6 +15,9 @@
 debug_gh = ndebug.create(__name__ + ':github')
 debug_gl = ndebug.create(__name__ + ':gitlab')
 
+# Add a mime type for wheels so asset upload doesn't fail
+mimetypes.add_type('application/x-wheel+zip', '.whl', False)
+
 
 class Base(object):
 
@@ -34,6 +38,11 @@ def post_release_changelog(
             cls, owner: str, repo: str, version: str, changelog: str) -> bool:
         raise NotImplementedError
 
+    @classmethod
+    def upload_dists(cls, owner: str, repo: str, version: str, path: str) -> bool:
+        # Skip on unsupported HVCS instead of raising error
+        return True
+
 
 class Github(Base):
     """Github helper class
@@ -74,6 +83,72 @@ def check_build_status(owner: str, repo: str, ref: str) -> bool:
             debug_gh('check_build_status: state={}'.format(response.json()['state']))
         return response.json()['state'] == 'success'
 
+    @classmethod
+    def create_release(cls, owner: str, repo: str, tag: str, changelog: str) -> bool:
+        """Create a new release
+
+        https://developer.github.com/v3/repos/releases/#create-a-release
+
+        :param owner: The owner namespace of the repository
+        :param repo: The repository name
+        :param tag: Tag to create release for
+        :param changelog: The release notes for this version
+
+        :return: Whether the request succeeded
+        """
+        response = requests.post(
+            f'{Github.API_URL}/repos/{owner}/{repo}/releases',
+            json={'tag_name': tag, 'body': changelog, 'draft': False, 'prerelease': False},
+            headers={'Authorization': 'token {}'.format(Github.token())}
+        )
+        debug_gh('Release creation: status={}'.format(response.status_code))
+
+        return response.status_code == 201
+
+    @classmethod
+    def get_release(cls, owner: str, repo: str, tag: str) -> int:
+        """Get a release by its tag name
+
+        https://developer.github.com/v3/repos/releases/#get-a-release-by-tag-name
+
+        :param owner: The owner namespace of the repository
+        :param repo: The repository name
+        :param tag: Tag to get release for
+
+        :return: ID of found release
+        """
+        response = requests.get(
+            f'{Github.API_URL}/repos/{owner}/{repo}/releases/tags/{tag}',
+            headers={'Authorization': 'token {}'.format(Github.token())}
+        )
+        debug_gh('Get release by tag: status={}, release_id={}'.format(
+                 response.status_code, response.json()['id']))
+
+        return response.json()['id']
+
+    @classmethod
+    def edit_release(cls, owner: str, repo: str, id: int, changelog: str) -> bool:
+        """Edit a release with updated change notes
+
+        https://developer.github.com/v3/repos/releases/#edit-a-release
+
+        :param owner: The owner namespace of the repository
+        :param repo: The repository name
+        :param id: ID of release to update
+        :param changelog: The release notes for this version
+
+        :return: Whether the request succeeded
+        """
+        response = requests.post(
+            f'{Github.API_URL}/repos/{owner}/{repo}/releases/{id}',
+            json={'body': changelog},
+            headers={'Authorization': 'token {}'.format(Github.token())}
+        )
+        debug_gh('Edit release: status={}, release_id={}'.format(
+                 response.status_code, id))
+
+        return response.status_code == 200
+
     @classmethod
     def post_release_changelog(
             cls, owner: str, repo: str, version: str, changelog: str) -> bool:
@@ -86,50 +161,80 @@ def post_release_changelog(
 
         :return: The status of the request
         """
-        url = '{domain}/repos/{owner}/{repo}/releases'
-        tag = 'v{0}'.format(version)
-        debug_gh('listing releases')
+        tag = f'v{version}'
+        debug_gh(f'Attempting to create release for {tag}')
+        success = Github.create_release(owner, repo, tag, changelog)
+
+        if not success:
+            debug_gh('Unsuccessful, looking for an existing release to update')
+            release_id = Github.get_release(owner, repo, tag)
+            debug_gh(f'Updating release {release_id}')
+            success = Github.edit_release(owner, repo, release_id, changelog)
+
+        return success
+
+    @classmethod
+    def upload_asset(
+            cls, owner: str, repo: str, release_id: int, file: str, label: str = None) -> bool:
+        """Upload an asset to an existing release
+
+        https://developer.github.com/v3/repos/releases/#upload-a-release-asset
+
+        :param owner: The owner namespace of the repository
+        :param repo: The repository name
+        :param release_id: ID of the release to upload to
+        :param file: Path of the file to upload
+        :param label: Custom label for this file
+
+        :return: The status of the request
+        """
+        url = 'https://uploads.github.com/repos/{owner}/{repo}/releases/{id}/assets'
+
         response = requests.post(
             url.format(
-                domain=Github.API_URL,
                 owner=owner,
-                repo=repo
+                repo=repo,
+                id=release_id
             ),
-            json={'tag_name': tag, 'body': changelog, 'draft': False, 'prerelease': False},
-            headers={'Authorization': 'token {}'.format(Github.token())}
+            params={'name': os.path.basename(file), 'label': label},
+            headers={
+                'Authorization': 'token {}'.format(Github.token()),
+                'Content-Type': mimetypes.guess_type(file, strict=False)[0]
+            },
+            data=open(file, 'rb').read()
         )
-        status, _ = response.status_code == 201, response.json()
-        debug_gh('response #1, status_code={}, status={}'.format(response.status_code, status))
-
-        if not status:
-            debug_gh('not status, getting tag', tag)
-            url = '{domain}/repos/{owner}/{repo}/releases/tags/{tag}'
-            response = requests.get(
-                url.format(
-                    domain=Github.API_URL,
-                    owner=owner,
-                    repo=repo,
-                    tag=tag
-                ),
-                headers={'Authorization': 'token {}'.format(Github.token())}
-            )
-            release_id = response.json()['id']
-            debug_gh('response #2, status_code={}'.format(response.status_code))
-            url = '{domain}/repos/{owner}/{repo}/releases/{id}'
-            debug_gh('getting release_id', release_id)
-            response = requests.post(
-                url.format(
-                    domain=Github.API_URL,
-                    owner=owner,
-                    repo=repo,
-                    id=release_id
-                ),
-                json={'tag_name': tag, 'body': changelog, 'draft': False, 'prerelease': False},
-                headers={'Authorization': 'token {}'.format(Github.token())}
-            )
-            status, _ = response.status_code == 200, response.json()
-            debug_gh('response #3, status_code={}, status={}'.format(response.status_code, status))
-        return status
+        debug_gh('Asset upload: url={}, status={}'.format(
+                 response.url, response.status_code))
+        debug_gh(response.json())
+        return response.status_code == 201
+
+    @classmethod
+    def upload_dists(cls, owner: str, repo: str, version: str, path: str) -> bool:
+        """Upload distributions to a release
+
+        :param owner: The owner namespace of the repository
+        :param repo: The repository name
+        :param version: Version to upload for
+        :param path: Path to the dist  directory
+
+        :return: The status of the request
+        """
+
+        # Find the release corresponding to this version
+        release_id = Github.get_release(owner, repo, f'v{version}')
+        if not release_id:
+            debug_gh('No release found to upload assets to')
+            return False
+
+        # Upload assets
+        one_or_more_failed = False
+        for file in os.listdir(path):
+            file_path = os.path.join(path, file)
+
+            if not Github.upload_asset(owner, repo, release_id, file_path):
+                one_or_more_failed = True
+
+        return not one_or_more_failed
 
 
 class Gitlab(Base):
@@ -250,6 +355,21 @@ def post_changelog(owner: str, repository: str, version: str, changelog: str) ->
     return get_hvcs().post_release_changelog(owner, repository, version, changelog)
 
 
+def upload_to_release(owner: str, repository: str, version: str, path: str) -> bool:
+    """
+    Posts the changelog to the current hvcs release API
+
+    :param owner: The owner of the repository
+    :param repository: The repository name
+    :param version: A string with the version to upload for
+    :param path: Path to dist directory
+
+    :return: Status of the request
+    """
+
+    return get_hvcs().upload_dists(owner, repository, version, path)
+
+
 def get_token() -> Optional[str]:
     """
     Returns the token for the current VCS

semantic_release/pypi.py

@@ -6,26 +6,24 @@
 
 
 def upload_to_pypi(
-        dists: str = 'sdist bdist_wheel',
         path: str = 'dist',
         username: str = None,
         password: str = None,
-        skip_existing: bool = False,
-        remove_dist: bool = True
+        skip_existing: bool = False
 ):
-    """Creates the wheel and uploads to pypi with twine.
+    """Upload wheels to PyPI with Twine.
 
-    :param dists: The dists string passed to setup.py. Default: 'bdist_wheel'
+    Wheels must already be created and stored at the given path.
+
+    :param path: Path to dist folder
     :param username: PyPI account username string
     :param password: PyPI account password string
     :param skip_existing: Continue uploading files if one already exists. (Only valid when
          uploading to PyPI. Other implementations may not support this.)
     """
     if username is None or password is None or username == "" or password == "":
         raise ImproperConfigurationError('Missing credentials for uploading')
-    if remove_dist:
-        run(f'rm -rf {path}')
-    run('python setup.py {}'.format(dists))
+
     run(
         "twine upload -u '{}' -p '{}' {} \"{}/*\"".format(
             username,
@@ -34,5 +32,3 @@ def upload_to_pypi(
             path
         )
     )
-    if remove_dist:
-        run(f'rm -rf {path}')