StableCascade Model Conversion Script

OnnxStack.Converter/README.md

@@ -0,0 +1,20 @@
+# OnnxStack.Converter
+
+## Requirements
+```bash
+pip install onnxruntime-directml
+pip install olive-ai[directml]
+python -m pip install -r requirements.txt
+```
+
+## Usage
+```bash
+convert.py --optimize --model_input '..\stable-diffusion-v1-5' --model_output '..\converted' --controlnet
+```
+`--optimize`  - Run the model optimization
+
+`--model_input`  - Safetensor model to convert
+
+`--model_output`  - Output for converted ONNX model (NOTE: This folder is deleted before each run)
+
+`--controlnet`  - Create a ControlNet enabled Unet model

OnnxStack.Converter/latent_consistency/.gitignore

@@ -0,0 +1,3 @@
+/footprints/
+/cache/
+/result_*.png

OnnxStack.Converter/latent_consistency/config.py

@@ -0,0 +1,8 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+
+vae_sample_size = 768
+unet_sample_size = 96
+cross_attention_dim = 768

OnnxStack.Converter/latent_consistency/config_controlnet.json

@@ -0,0 +1,124 @@
+{
+  "input_model": {
+      "type": "PyTorchModel",
+      "config": {
+          "model_path": "SimianLuo/LCM_Dreamshaper_v7",
+          "model_loader": "controlnet_unet_load",
+          "model_script": "models.py",
+          "io_config": {
+              "input_names": [ "sample", "timestep", "encoder_hidden_states", "down_block_0_additional_residual", "down_block_1_additional_residual", "down_block_2_additional_residual", "down_block_3_additional_residual", "down_block_4_additional_residual", "down_block_5_additional_residual", "down_block_6_additional_residual", "down_block_7_additional_residual", "down_block_8_additional_residual", "down_block_9_additional_residual", "down_block_10_additional_residual", "down_block_11_additional_residual", "mid_block_additional_residual", "return_dict" ],
+              "output_names": [ "out_sample" ],
+              "dynamic_axes": {
+                  "sample": {"0": "unet_sample_batch", "1": "unet_sample_channels", "2": "unet_sample_height", "3": "unet_sample_width"},
+                  "timestep": {"0": "unet_time_batch"},
+                  "encoder_hidden_states": {"0": "unet_hidden_batch", "1": "unet_hidden_sequence"},
+				  "timestep_cond": { "0": "batch_size"  },
+                  "down_block_0_additional_residual": {"0": "cnet_db0_batch", "1": "cnet_db0_channels", "2": "cnet_db0_height", "3": "cnet_db0_width"},
+                  "down_block_1_additional_residual": {"0": "cnet_db1_batch", "1": "cnet_db1_channels", "2": "cnet_db1_height", "3": "cnet_db1_width"},
+                  "down_block_2_additional_residual": {"0": "cnet_db2_batch", "1": "cnet_db2_channels", "2": "cnet_db2_height", "3": "cnet_db2_width"},
+                  "down_block_3_additional_residual": {"0": "cnet_db3_batch", "1": "cnet_db3_channels", "2": "cnet_db3_height2", "3": "cnet_db3_width2"},
+                  "down_block_4_additional_residual": {"0": "cnet_db4_batch", "1": "cnet_db4_channels", "2": "cnet_db4_height2", "3": "cnet_db4_width2"},
+                  "down_block_5_additional_residual": {"0": "cnet_db5_batch", "1": "cnet_db5_channels", "2": "cnet_db5_height2", "3": "cnet_db5_width2"},
+                  "down_block_6_additional_residual": {"0": "cnet_db6_batch", "1": "cnet_db6_channels", "2": "cnet_db6_height4", "3": "cnet_db6_width4"},
+                  "down_block_7_additional_residual": {"0": "cnet_db7_batch", "1": "cnet_db7_channels", "2": "cnet_db7_height4", "3": "cnet_db7_width4"},
+                  "down_block_8_additional_residual": {"0": "cnet_db8_batch", "1": "cnet_db8_channels", "2": "cnet_db8_height4", "3": "cnet_db8_width4"},
+                  "down_block_9_additional_residual": {"0": "cnet_db9_batch", "1": "cnet_db9_channels", "2": "cnet_db9_height8", "3": "cnet_db9_width8"},
+                  "down_block_10_additional_residual": {"0": "cnet_db10_batch", "1": "cnet_db10_channels", "2": "cnet_db10_height8", "3": "cnet_db10_width8"},
+                  "down_block_11_additional_residual": {"0": "cnet_db11_batch", "1": "cnet_db11_channels", "2": "cnet_db11_height8", "3": "cnet_db11_width8"},
+                  "mid_block_additional_residual": {"0": "cnet_mbar_batch", "1": "cnet_mbar_channels", "2": "cnet_mbar_height8", "3": "cnet_mbar_width8"}
+              }
+          },
+          "dummy_inputs_func": "controlnet_unet_conversion_inputs"
+      }
+  },
+  "systems": {
+      "local_system": {
+          "type": "LocalSystem",
+          "config": {
+                "accelerators": [
+                    {
+                        "device": "gpu",
+                        "execution_providers": [
+                            "DmlExecutionProvider"
+                        ]
+                    }
+                ]
+          }
+      }
+  },
+  "evaluators": {
+      "common_evaluator": {
+          "metrics": [
+              {
+                  "name": "latency",
+                  "type": "latency",
+                  "sub_types": [{"name": "avg"}],
+                  "user_config": {
+                      "user_script": "models.py",
+                      "dataloader_func": "controlnet_unet_data_loader",
+                      "batch_size": 2
+                  }
+              }
+          ]
+      }
+  },
+  "passes": {
+      "convert": {
+          "type": "OnnxConversion",
+          "config": {
+              "target_opset": 14,
+              "save_as_external_data": true,
+              "all_tensors_to_one_file": true,
+              "external_data_name": "weights.pb"
+          }
+      },
+      "optimize": {
+            "type": "OrtTransformersOptimization",
+            "config": {
+                "model_type": "unet",
+                "opt_level": 0,
+                "float16": true,
+                "use_gpu": true,
+                "keep_io_types": false,
+                "optimization_options": {
+                    "enable_gelu": true,
+                    "enable_layer_norm": true,
+                    "enable_attention": true,
+                    "use_multi_head_attention": true,
+                    "enable_skip_layer_norm": false,
+                    "enable_embed_layer_norm": true,
+                    "enable_bias_skip_layer_norm": false,
+                    "enable_bias_gelu": true,
+                    "enable_gelu_approximation": false,
+                    "enable_qordered_matmul": false,
+                    "enable_shape_inference": true,
+                    "enable_gemm_fast_gelu": false,
+                    "enable_nhwc_conv": false,
+                    "enable_group_norm": true,
+                    "enable_bias_splitgelu": false,
+                    "enable_packed_qkv": true,
+                    "enable_packed_kv": true,
+                    "enable_bias_add": false,
+                    "group_norm_channels_last": false
+                },
+                "force_fp32_ops": ["RandomNormalLike"],
+                "force_fp16_inputs": {
+                    "GroupNorm": [0, 1, 2]
+                }
+            }
+      }
+    },
+    "pass_flows": [
+        ["convert", "optimize"]
+    ],
+    "engine": {
+        "log_severity_level": 0,
+        "evaluator": "common_evaluator",
+        "evaluate_input_model": false,
+        "host": "local_system",
+        "target": "local_system",
+        "cache_dir": "cache",
+        "output_name": "controlnet",
+        "output_dir": "footprints"
+    }
+}

OnnxStack.Converter/latent_consistency/config_safety_checker.json

@@ -0,0 +1,124 @@
+{
+    "input_model": {
+        "type": "PyTorchModel",
+        "config": {
+            "model_path": "SimianLuo/LCM_Dreamshaper_v7",
+            "model_loader": "safety_checker_load",
+            "model_script": "models.py",
+            "io_config": {
+                "input_names": [ "clip_input", "images" ],
+                "output_names": [ "out_images", "has_nsfw_concepts" ],
+                "dynamic_axes": {
+                    "clip_input": { "0": "batch", "1": "channels", "2": "height", "3": "width" },
+                    "images": { "0": "batch", "1": "height", "2": "width", "3": "channels" }
+                }
+            },
+            "dummy_inputs_func": "safety_checker_conversion_inputs"
+        }
+    },
+    "systems": {
+        "local_system": {
+            "type": "LocalSystem",
+            "config": {
+                "accelerators": [
+                    {
+                        "device": "gpu",
+                        "execution_providers": [
+                            "DmlExecutionProvider"
+                        ]
+                    }
+                ]
+            }
+        }
+    },
+    "evaluators": {
+        "common_evaluator": {
+            "metrics": [
+                {
+                    "name": "latency",
+                    "type": "latency",
+                    "sub_types": [{"name": "avg"}],
+                    "user_config": {
+                        "user_script": "models.py",
+                        "dataloader_func": "safety_checker_data_loader",
+                        "batch_size": 1
+                    }
+                }
+            ]
+        }
+    },
+    "passes": {
+        "convert": {
+            "type": "OnnxConversion",
+            "config": {
+                "target_opset": 14
+            }
+        },
+        "ov_convert": {
+            "type": "OpenVINOConversion",
+            "config": {
+                "user_script": "models.py",
+                "example_input_func": "safety_checker_conversion_inputs",
+                "output_model": "safety_checker"
+            }
+        },
+        "optimize": {
+            "type": "OrtTransformersOptimization",
+            "config": {
+                "model_type": "unet",
+                "opt_level": 0,
+                "float16": true,
+                "use_gpu": true,
+                "keep_io_types": false,
+                "optimization_options": {
+                    "enable_gelu": true,
+                    "enable_layer_norm": true,
+                    "enable_attention": true,
+                    "use_multi_head_attention": true,
+                    "enable_skip_layer_norm": false,
+                    "enable_embed_layer_norm": true,
+                    "enable_bias_skip_layer_norm": false,
+                    "enable_bias_gelu": true,
+                    "enable_gelu_approximation": false,
+                    "enable_qordered_matmul": false,
+                    "enable_shape_inference": true,
+                    "enable_gemm_fast_gelu": false,
+                    "enable_nhwc_conv": false,
+                    "enable_group_norm": true,
+                    "enable_bias_splitgelu": false,
+                    "enable_packed_qkv": true,
+                    "enable_packed_kv": true,
+                    "enable_bias_add": false,
+                    "group_norm_channels_last": false
+                },
+                "force_fp32_ops": ["RandomNormalLike"],
+                "force_fp16_inputs": {
+                    "GroupNorm": [0, 1, 2]
+                }
+            }
+        },
+        "optimize_cuda": {
+            "type": "OrtTransformersOptimization",
+            "config": {
+                "model_type": "unet",
+                "opt_level": 0,
+                "float16": true,
+                "use_gpu": true,
+                "keep_io_types": false
+            }
+        }
+    },
+    "pass_flows": [
+        ["convert", "optimize"]
+    ],
+    "engine": {
+        "log_severity_level": 0,
+        "evaluator": "common_evaluator",
+        "evaluate_input_model": false,
+        "host": "local_system",
+        "target": "local_system",
+        "cache_dir": "cache",
+        "output_name": "safety_checker",
+        "output_dir": "footprints"
+    }
+}