[PyPI publishing] feat: automate the process of pypi publication to s…

…ome extent. (#7270)

* feat: automate the process of pypi publication to some extent.

* utility to fetch the latest release branch

* correct package name.

.github/workflows/notify_slack_about_release.yml

@@ -0,0 +1,23 @@
+name: Notify Slack about a release
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.8'
+
+    - name: Notify Slack about the release
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      run: pip install requests && python utils/notify_slack_about_release.py

.github/workflows/pypi_publish.yaml

@@ -0,0 +1,79 @@
+# Adapted from https://blog.deepjyoti30.dev/pypi-release-github-action
+
+name: PyPI release
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  find-and-checkout-latest-branch:
+    runs-on: ubuntu-latest
+    outputs:
+      latest_branch: ${{ steps.set_latest_branch.outputs.latest_branch }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Fetch latest branch
+        id: fetch_latest_branch
+        run: |
+          pip install -U requests packaging
+          LATEST_BRANCH=$(python utils/fetch_latest_release_branch.py)
+          echo "Latest branch: $LATEST_BRANCH"
+          echo "latest_branch=$LATEST_BRANCH" >> $GITHUB_ENV
+          
+      - name: Set latest branch output
+        id: set_latest_branch
+        run: echo "::set-output name=latest_branch::${{ env.latest_branch }}"
+
+  release:
+    needs: find-and-checkout-latest-branch
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.find-and-checkout-latest-branch.outputs.latest_branch }}
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -U setuptools wheel twine
+      
+      - name: Build the dist files
+        run: python setup.py bdist_wheel && python setup.py sdist
+
+      - name: Publish to the test PyPI
+        env:
+          TWINE_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }}
+        run: twine upload dist/* -r pypitest --repository-url=https://test.pypi.org/legacy/    
+
+      - name: Test installing diffusers and importing
+        run: |
+          pip install diffusers && pip uninstall diffusers -y
+          pip install -i https://testpypi.python.org/pypi diffusers
+          python -c "from diffusers import __version__; print(__version__)"
+          python -c "from diffusers import DiffusionPipeline; pipe = DiffusionPipeline.from_pretrained('fusing/unet-ldm-dummy-update'); pipe()"
+          python -c "from diffusers import DiffusionPipeline; pipe = DiffusionPipeline.from_pretrained('hf-internal-testing/tiny-stable-diffusion-pipe', safety_checker=None); pipe('ah suh du')"
+          python -c "from diffusers import *"
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: twine upload dist/* -r pypi

utils/fetch_latest_release_branch.py

@@ -0,0 +1,68 @@
+# coding=utf-8
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import requests
+from packaging.version import parse
+
+
+# GitHub repository details
+USER = "huggingface"
+REPO = "diffusers"
+
+
+def fetch_all_branches(user, repo):
+    branches = []  # List to store all branches
+    page = 1  # Start from first page
+    while True:
+        # Make a request to the GitHub API for the branches
+        response = requests.get(f"https://api.github.com/repos/{user}/{repo}/branches", params={"page": page})
+
+        # Check if the request was successful
+        if response.status_code == 200:
+            # Add the branches from the current page to the list
+            branches.extend([branch["name"] for branch in response.json()])
+
+            # Check if there is a 'next' link for pagination
+            if "next" in response.links:
+                page += 1  # Move to the next page
+            else:
+                break  # Exit loop if there is no next page
+        else:
+            print("Failed to retrieve branches:", response.status_code)
+            break
+
+    return branches
+
+
+def main():
+    # Fetch all branches
+    branches = fetch_all_branches(USER, REPO)
+
+    # Filter branches.
+    # print(f"Total branches: {len(branches)}")
+    filtered_branches = []
+    for branch in branches:
+        if branch.startswith("v") and ("-release" in branch or "-patch" in branch):
+            filtered_branches.append(branch)
+            # print(f"Filtered: {branch}")
+
+    sorted_branches = sorted(filtered_branches, key=lambda x: parse(x.split("-")[0][1:]), reverse=True)
+    latest_branch = sorted_branches[0]
+    # print(f"Latest branch: {latest_branch}")
+    return latest_branch
+
+
+if __name__ == "__main__":
+    print(main())

utils/notify_slack_about_release.py

@@ -0,0 +1,80 @@
+# coding=utf-8
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import requests
+
+
+# Configuration
+LIBRARY_NAME = "diffusers"
+GITHUB_REPO = "huggingface/diffusers"
+SLACK_WEBHOOK_URL = os.getenv("SLACK_WEBHOOK_URL")
+
+
+def check_pypi_for_latest_release(library_name):
+    """Check PyPI for the latest release of the library."""
+    response = requests.get(f"https://pypi.org/pypi/{library_name}/json")
+    if response.status_code == 200:
+        data = response.json()
+        return data["info"]["version"]
+    else:
+        print("Failed to fetch library details from PyPI.")
+        return None
+
+
+def get_github_release_info(github_repo):
+    """Fetch the latest release info from GitHub."""
+    url = f"https://api.github.com/repos/{github_repo}/releases/latest"
+    response = requests.get(url)
+
+    if response.status_code == 200:
+        data = response.json()
+        return {"tag_name": data["tag_name"], "url": data["html_url"], "release_time": data["published_at"]}
+
+    else:
+        print("Failed to fetch release info from GitHub.")
+        return None
+
+
+def notify_slack(webhook_url, library_name, version, release_info):
+    """Send a notification to a Slack channel."""
+    message = (
+        f"🚀 New release for {library_name} available: version **{version}** 🎉\n"
+        f"📜 Release Notes: {release_info['url']}\n"
+        f"⏱️ Release time: {release_info['release_time']}"
+    )
+    payload = {"text": message}
+    response = requests.post(webhook_url, json=payload)
+
+    if response.status_code == 200:
+        print("Notification sent to Slack successfully.")
+    else:
+        print("Failed to send notification to Slack.")
+
+
+def main():
+    latest_version = check_pypi_for_latest_release(LIBRARY_NAME)
+    release_info = get_github_release_info(GITHUB_REPO)
+    parsed_version = release_info["tag_name"].replace("v", "")
+
+    if latest_version and release_info and latest_version == parsed_version:
+        notify_slack(SLACK_WEBHOOK_URL, LIBRARY_NAME, latest_version, release_info)
+    else:
+        raise ValueError("There were some problems.")
+
+
+if __name__ == "__main__":
+    main()