From 4c261e985fc8f49bc3ec89db3d84e934b3da5e63 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Thu, 4 Apr 2024 16:28:28 +0530 Subject: [PATCH 1/3] add utilities for updating diffusers pipeline metadata. --- .github/workflows/update_metadata.yml | 31 ++++++++ utils/update_metadata.py | 103 ++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 .github/workflows/update_metadata.yml create mode 100644 utils/update_metadata.py diff --git a/.github/workflows/update_metadata.yml b/.github/workflows/update_metadata.yml new file mode 100644 index 000000000000..ce1c56474aaf --- /dev/null +++ b/.github/workflows/update_metadata.yml @@ -0,0 +1,31 @@ + +name: Update Diffusers metadata + +on: + workflow_dispatch: + push: + branches: + - main + - update_diffusers_metadata* + +jobs: + update_metadata: + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v3 + + - name: Setup environment + run: | + pip install --upgrade pip + pip install datasets pandas + pip install .[torch] + + - name: Update metadata + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.DIFFUSERS_BOT_TOKEN }} + run: | + python utils/update_metadata.py --commit_sha ${{ github.sha }} diff --git a/utils/update_metadata.py b/utils/update_metadata.py new file mode 100644 index 000000000000..0ddbd22b5f4b --- /dev/null +++ b/utils/update_metadata.py @@ -0,0 +1,103 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Utility that updates the metadata of the Diffusers library in the repository `huggingface/diffusers-metadata`. + +Usage for an update (as used by the GitHub action `update_metadata`): + +```bash +python utils/update_metadata.py +``` + +Script modified from: +https://github.com/huggingface/transformers/blob/main/utils/update_metadata.py +""" +import argparse +import os +import tempfile +import pandas as pd + +from datasets import Dataset +from huggingface_hub import upload_folder + +from diffusers.pipelines.auto_pipeline import ( + AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, + AUTO_INPAINT_PIPELINES_MAPPING, + AUTO_TEXT2IMAGE_PIPELINES_MAPPING, +) + + +def get_supported_pipeline_table() -> dict: + """ + Generates a dictionary containing the supported auto classes for each pipeline type, + using the content of the auto modules. + """ + # All supported pipelines for automatic mapping. + all_supported_pipeline_classes = [ + (class_name.__name__, "text-to-image", "AutoPipelineForText2Image") for _, class_name in AUTO_TEXT2IMAGE_PIPELINES_MAPPING.items() + ] + all_supported_pipeline_classes += [ + (class_name.__name__, "image-to-image", "AutoPipelineForImage2Image") for _, class_name in AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.items() + ] + all_supported_pipeline_classes += [ + (class_name.__name__, "image-to-image", "AutoPipelineForInpainting") for _, class_name in AUTO_INPAINT_PIPELINES_MAPPING.items() + ] + all_supported_pipeline_classes.sort(key=lambda x: x[0]) + all_supported_pipeline_classes = list(set(all_supported_pipeline_classes)) + + data = {} + data["pipeline_class"] = [sample[0] for sample in all_supported_pipeline_classes] + data["pipeline_tag"] = [sample[1] for sample in all_supported_pipeline_classes] + data["auto_class"] = [sample[2] for sample in all_supported_pipeline_classes] + + return data + + +def update_metadata(commit_sha: str): + """ + Update the metadata for the Diffusers repo in `huggingface/diffusers-metadata`. + + Args: + commit_sha (`str`): The commit SHA on Diffusers corresponding to this update. + """ + pipelines_table = get_supported_pipeline_table() + pipelines_table = pd.DataFrame(pipelines_table) + pipelines_dataset = Dataset.from_pandas(pipelines_table) + + with tempfile.TemporaryDirectory() as tmp_dir: + pipelines_dataset.to_json(os.path.join(tmp_dir, "pipeline_tags.json")) + + if commit_sha is not None: + commit_message = ( + f"Update with commit {commit_sha}\n\nSee: " + f"https://github.com/huggingface/diffusers/commit/{commit_sha}" + ) + else: + commit_message = "Update" + + upload_folder( + repo_id="huggingface/diffusers-metadata", + folder_path=tmp_dir, + repo_type="dataset", + commit_message=commit_message, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--commit_sha", default=None, type=str, help="The sha of the commit going with this update.") + args = parser.parse_args() + + update_metadata(args.commit_sha) From b9047aa9cc8cc3b640c3ad8ded413143d94d6afa Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Thu, 4 Apr 2024 16:30:37 +0530 Subject: [PATCH 2/3] style --- utils/update_metadata.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/utils/update_metadata.py b/utils/update_metadata.py index 0ddbd22b5f4b..840e4be78423 100644 --- a/utils/update_metadata.py +++ b/utils/update_metadata.py @@ -27,8 +27,8 @@ import argparse import os import tempfile -import pandas as pd +import pandas as pd from datasets import Dataset from huggingface_hub import upload_folder @@ -46,13 +46,16 @@ def get_supported_pipeline_table() -> dict: """ # All supported pipelines for automatic mapping. all_supported_pipeline_classes = [ - (class_name.__name__, "text-to-image", "AutoPipelineForText2Image") for _, class_name in AUTO_TEXT2IMAGE_PIPELINES_MAPPING.items() + (class_name.__name__, "text-to-image", "AutoPipelineForText2Image") + for _, class_name in AUTO_TEXT2IMAGE_PIPELINES_MAPPING.items() ] all_supported_pipeline_classes += [ - (class_name.__name__, "image-to-image", "AutoPipelineForImage2Image") for _, class_name in AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.items() + (class_name.__name__, "image-to-image", "AutoPipelineForImage2Image") + for _, class_name in AUTO_IMAGE2IMAGE_PIPELINES_MAPPING.items() ] all_supported_pipeline_classes += [ - (class_name.__name__, "image-to-image", "AutoPipelineForInpainting") for _, class_name in AUTO_INPAINT_PIPELINES_MAPPING.items() + (class_name.__name__, "image-to-image", "AutoPipelineForInpainting") + for _, class_name in AUTO_INPAINT_PIPELINES_MAPPING.items() ] all_supported_pipeline_classes.sort(key=lambda x: x[0]) all_supported_pipeline_classes = list(set(all_supported_pipeline_classes)) @@ -61,7 +64,7 @@ def get_supported_pipeline_table() -> dict: data["pipeline_class"] = [sample[0] for sample in all_supported_pipeline_classes] data["pipeline_tag"] = [sample[1] for sample in all_supported_pipeline_classes] data["auto_class"] = [sample[2] for sample in all_supported_pipeline_classes] - + return data From 7001370cc492716edb78823ea84f3cb98c31be3d Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Thu, 4 Apr 2024 16:34:38 +0530 Subject: [PATCH 3/3] remove first empty line --- .github/workflows/update_metadata.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/update_metadata.yml b/.github/workflows/update_metadata.yml index ce1c56474aaf..33d162ef8d1f 100644 --- a/.github/workflows/update_metadata.yml +++ b/.github/workflows/update_metadata.yml @@ -1,4 +1,3 @@ - name: Update Diffusers metadata on: