cleanup and enhancement

01.sh

@@ -1,13 +1,36 @@
 #!/bin/bash
 source /sl_folder.sh
 
-SD01_DIR="${BASE_DIR}/01-easy-diffusion"
+export PATH="/home/abc/miniconda3/bin:$PATH"
+export SD01_DIR=${BASE_DIR}/01-easy-diffusion
 
+#clean conda env
+if [ "$active_clean" = "1" ]; then
+    echo "-------------------------------------"
+    echo "Cleaning venv"
+    rm -rf ${SD01_DIR}/conda-env
+    rm -rf ${SD01_DIR}/installer_files
+    export active_clean=0
+    echo "Done!"
+    echo -e "-------------------------------------\n"
+fi
+
+#Create Conda Env
+if [ ! -d ${SD01_DIR}/conda-env ]; then
+    conda create -p ${SD01_DIR}/conda-env -y
+fi
+
+#active env and install python
+source activate ${SD01_DIR}/conda-env
+conda install -n base conda-libmamba-solver -y
+conda install -c python=3.11 pip --solver=libmamba -y
+
+#create 'models' folders
 mkdir -p ${SD01_DIR}/{models,version,plugins/ui,scripts}
 cd ${SD01_DIR}
 
+#install manager plugin
 if [ ! -f ${SD01_DIR}/plugins/ui/plugin-manager.plugin.js ]; then
-    mkdir -p ${SD01_DIR}/plugins/ui/
     curl -L https://raw.githubusercontent.com/patriceac/Easy-Diffusion-Plugins/main/plugin-manager.plugin.js --output ${SD01_DIR}/plugins/ui/plugin-manager.plugin.js
 fi
 
@@ -24,10 +47,12 @@ sl_folder ${SD01_DIR}/models gfpgan ${BASE_DIR}/models gfpgan
 
 sl_folder ${HOME} "Stable Diffusion UI" ${BASE_DIR}/outputs 01-Easy-Diffusion
 
+#copy default parameters if they don't exists
 if [ ! -f "$SD01_DIR/config.yaml" ]; then
     cp -v "${SD_INSTALL_DIR}/parameters/01.txt" "$SD01_DIR/config.yaml"
 fi
 
+#download installer if it isn't present
 if [ ! -f "$SD01_DIR/start.sh" ]; then
     curl -L https://github.com/easydiffusion/easydiffusion/releases/download/v3.0.2/Easy-Diffusion-Linux.zip --output edui.zip
     unzip edui.zip
@@ -36,7 +61,6 @@ if [ ! -f "$SD01_DIR/start.sh" ]; then
     rm -f edui.zip
 fi
 
-#chown -R diffusion:users ${BASE_DIR}
-
+#run easy-diffusion
 cd $SD01_DIR
 bash start.sh

02.forge.sh

@@ -2,22 +2,19 @@
 source /sl_folder.sh
 
 export PATH="/home/abc/miniconda3/bin:$PATH"
-export active_clean=0
+export SD02_DIR=${BASE_DIR}/02-sd-webui
 
 # disable the use of a python venv
 export venv_dir="-"
 
 # Install or update Stable-Diffusion-WebUI
 mkdir -p ${SD02_DIR}
 
+# Create Conda virtual env
 if [ ! -d ${SD02_DIR}/conda-env ]; then
     conda create -p ${SD02_DIR}/conda-env -y
 fi
 
-source activate ${SD02_DIR}/conda-env
-conda install -n base conda-libmamba-solver -y
-conda install -c git python=3.11 pip --solver=libmamba -y
-
 if [ ! -d ${SD02_DIR}/forge ]; then
     git clone https://github.com/lllyasviel/stable-diffusion-webui-forge.git ${SD02_DIR}/forge
 fi
@@ -41,24 +38,32 @@ fi
 
 #clean conda env
 if [ "$active_clean" = "1" ]; then
+    echo "-------------------------------------"
+    echo "Cleaning venv"
     conda deactivate
-    conda remove -p ${SD02_DIR}/conda-env --all -y
+    rm -rf ${SD02_DIR}/conda-env
     conda create -p ${SD02_DIR}/conda-env -y
-    source activate ${SD02_DIR}/conda-env
+    export active_clean=0
+    echo "Done!"
+    echo -e "-------------------------------------\n"
 fi
-conda install -c conda-forge git python=3.11 pip gcc gxx libcurand --solver=libmamba -y
+
+#activate conda env + install base tools
+source activate ${SD02_DIR}/conda-env
+conda install -n base conda-libmamba-solver -y
+conda install -c conda-forge python=3.11 pip gcc gxx libcurand --solver=libmamba -y
 
 if [ ! -f "$SD02_DIR/parameters.forge.txt" ]; then
     cp -v "/opt/sd-install/parameters/02.forge.txt" "$SD02_DIR/parameters.forge.txt"
 fi
 
+#install dependencies 
 pip install --upgrade pip
 pip install coloredlogs flatbuffers numpy packaging protobuf==3.20.3 sympy
 pip install packaging
 pip install onnxruntime-gpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
 pip install insightface
 pip install basicsr
-pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu121
 pip install xformers --index-url https://download.pytorch.org/whl/cu121
 
 # Merge Models, vae, lora, and hypernetworks, and outputs
@@ -77,6 +82,7 @@ sl_folder ${SD02_DIR}/forge/models ControlNet ${BASE_DIR}/models controlnet
 
 sl_folder ${SD02_DIR}/forge output ${BASE_DIR}/outputs 02-sd-webui
 
+# Run webUI
 echo "Run Stable-Diffusion-WebUI-forge"
 cd ${SD02_DIR}/forge
 CMD="bash webui.sh"

02.sh

@@ -2,24 +2,22 @@
 source /sl_folder.sh
 
 export PATH="/home/abc/miniconda3/bin:$PATH"
-export active_clean=0
+export SD02_DIR=${BASE_DIR}/02-sd-webui
 
 # disable the use of a python venv
 export venv_dir="-"
 
 # Install or update Stable-Diffusion-WebUI
 mkdir -p ${SD02_DIR}
 
+# create conda env if needed
 if [ ! -d ${SD02_DIR}/conda-env ]; then
     conda create -p ${SD02_DIR}/conda-env -y
 fi
 
-source activate ${SD02_DIR}/conda-env
-conda install -n base conda-libmamba-solver -y
-conda install -c git python=3.11 pip --solver=libmamba -y
-
+# clone repository
 if [ ! -d ${SD02_DIR}/webui ]; then
-    git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui ${SD02_DIR}/webui
+    git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui.git ${SD02_DIR}/webui
 fi
 
 cd ${SD02_DIR}/webui
@@ -41,24 +39,33 @@ fi
 
 #clean conda env
 if [ "$active_clean" = "1" ]; then
+    echo "-------------------------------------"
+    echo "Cleaning venv"
     conda deactivate
-    conda remove -p ${SD02_DIR}/conda-env --all -y
+    rm -rf ${SD02_DIR}/conda-env
     conda create -p ${SD02_DIR}/conda-env -y
-    source activate ${SD02_DIR}/conda-env
+    export active_clean=0
+    echo "Done!"
+    echo -e "-------------------------------------\n"
 fi
-conda install -c conda-forge git python=3.11 pip gcc gxx libcurand --solver=libmamba -y
 
+# activate conda env and install base tools
+source activate ${SD02_DIR}/conda-env
+conda install -n base conda-libmamba-solver -y
+conda install -c conda-forge python=3.11 pip gcc gxx libcurand --solver=libmamba -y
+
+#copy default parameters if absent
 if [ ! -f "$SD02_DIR/parameters.txt" ]; then
     cp -v "/opt/sd-install/parameters/02.txt" "$SD02_DIR/parameters.txt"
 fi
 
+# install dependencies
 pip install --upgrade pip
 pip install coloredlogs flatbuffers numpy packaging protobuf==3.20.3 sympy
 pip install packaging
 pip install onnxruntime-gpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
 pip install insightface
 pip install basicsr
-pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu121
 pip install xformers --index-url https://download.pytorch.org/whl/cu121
 
 
@@ -78,6 +85,7 @@ sl_folder ${SD02_DIR}/webui/models ControlNet ${BASE_DIR}/models controlnet
 
 sl_folder ${SD02_DIR}/webui outputs ${BASE_DIR}/outputs 02-sd-webui
 
+# run webUI
 echo "Run Stable-Diffusion-WebUI"
 cd ${SD02_DIR}/webui
 CMD="bash webui.sh"

03.sh

@@ -3,21 +3,35 @@ source /sl_folder.sh
 
 export PATH="/home/abc/miniconda3/bin:$PATH"
 export use_venv=0
+export SD03_DIR=${BASE_DIR}/03-invokeai
 
 mkdir -p "$SD03_DIR"
 mkdir -p /config/outputs/03-InvokeAI
 
+# copy default parameters if absent
 if [ ! -f "$SD03_DIR/parameters.txt" ]; then
     cp -v "${SD_INSTALL_DIR}/parameters/03.txt" "$SD03_DIR/parameters.txt"
 fi
 
+#clean conda env
+if [ "$active_clean" = "1" ]; then
+    echo "-------------------------------------"
+    echo "Cleaning venv"
+    rm -rf ${SD03_DIR}/env
+    export active_clean=0
+    echo "Done!"
+    echo -e "-------------------------------------\n"
+fi
+
+# create conda env if absent
 if [ ! -d ${SD03_DIR}/env ]; then
     conda create -p ${SD03_DIR}/env -y
 fi
 
+# activate conda env and install basic tools
 source activate ${SD03_DIR}/env
 conda install -n base conda-libmamba-solver -y
-conda install -c conda-forge git python=3.10 --solver=libmamba -y
+conda install -c conda-forge python=3.10 --solver=libmamba -y
 
 cd ${SD03_DIR}
 
@@ -31,8 +45,7 @@ fi
 pip install --use-pep517 --upgrade InvokeAI
 invokeai-configure --yes --root ${SD03_DIR}/invokeai --skip-sd-weights
 
-# pip install ruamel.yaml==0.17.40 pillow==10.0.1 pypatchmatch
-
+# launch WebUI
 CMD="invokeai-web"
 while IFS= read -r param; do
     if [[ $param != \#* ]]; then

04.sh

@@ -2,71 +2,77 @@
 source /sl_folder.sh
 
 export PATH="/home/abc/miniconda3/bin:$PATH"
+export SD04_DIR=${BASE_DIR}/04-SD-Next
 export use_venv=1
 
 echo "Install and run SD-Next"
 
 mkdir -p ${SD04_DIR}
 
-if [ ! -d ${SD04_DIR}/env ]; then
-    conda create -p ${SD04_DIR}/env -y
-fi
-
-source activate ${SD04_DIR}/env
-conda install -n base conda-libmamba-solver -y
-conda install -c conda-forge git python=3.11 pip gxx libcurand gcc gxx onnx --solver=libmamba -y
-conda install -c nvidia cuda-cudart --solver=libmamba -y
-
+#clone main repository
 if [ ! -d ${SD04_DIR}/webui ]; then
     git clone https://github.com/vladmandic/automatic ${SD04_DIR}/webui
 fi
 
 cd ${SD04_DIR}/webui
 
-if [ -d "${SD04_DIR}/webui/venv" ]; then
-    # check if remote is ahead of local
-    # https://stackoverflow.com/a/25109122/1469797
-    if [ "$CLEAN_ENV" != "true" ] && [ $(git rev-parse HEAD) = $(git ls-remote $(git rev-parse --abbrev-ref @{u} | \
-    sed 's/\// /g') | cut -f1) ]; then
-         echo "Local branch up-to-date, keeping existing venv"
-      else
-        if [ "$CLEAN_ENV" = "true" ]; then
-          echo "Forced wiping venv for clean packages install"
-        else
-          echo "Remote branch is ahead. Wiping venv for clean packages install"
-        fi
-        rm -rf ${SD04_DIR}/webui/venv
-        git pull -X ours
+# check if remote is ahead of local
+# https://stackoverflow.com/a/25109122/1469797
+if [ "$CLEAN_ENV" != "true" ] && [ $(git rev-parse HEAD) = $(git ls-remote $(git rev-parse --abbrev-ref @{u} | \
+  sed 's/\// /g') | cut -f1) ]; then
+    echo "Local branch up-to-date, keeping existing venv"
+    else
+    if [ "$CLEAN_ENV" = "true" ]; then
+      echo "Forced wiping venv for clean packages install"
+    else
+      echo "Remote branch is ahead. Wiping venv for clean packages install"
     fi
+  export active_clean=1
+  git pull -X ours
 fi
 
-if [ ! -f "$SD04_DIR/parameters.txt" ]; then
-    cp -v "${SD_INSTALL_DIR}/parameters/04.txt" "$SD04_DIR/parameters.txt"
+#clean virtual env
+if [ "$active_clean" = "1" ]; then
+    echo "-------------------------------------"
+    echo "Cleaning venv"
+    rm -rf ${SD04_DIR}/env
+    rm -rf ${SD04_DIR}/webui/venv
+    export active_clean=0
+    echo "Done!"
+    echo -e "-------------------------------------\n"
 fi
-# Load updated malloc to fix memory leak
-# https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/6850#issuecomment-1432435503
-# https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/6722
-# https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/6722#discussioncomment-6161333
-if [[ "${OSTYPE}" == "linux"* ]] && [[ -z "${NO_TCMALLOC}" ]] && [[ -z "${LD_PRELOAD}" ]]; then
-    TCMALLOC="$(ldconfig -p | grep -Po "libtcmalloc_minimal.so.\d" | head -n 1)"
-    if [[ ! -z "${TCMALLOC}" ]]; then
-       echo "Using TCMalloc: ${TCMALLOC}"
-       export LD_PRELOAD="${TCMALLOC}"
-    fi
+
+#create conda env if needed
+if [ ! -d ${SD04_DIR}/env ]; then
+    conda create -p ${SD04_DIR}/env -y
 fi
 
+#activate and install basic tools
+source activate ${SD04_DIR}/env
+conda install -n base conda-libmamba-solver -y
+conda install -c conda-forge python=3.11 pip gcc gxx --solver=libmamba -y
+
 # Create venv
 if [ ! -d ${SD04_DIR}/webui/venv ]; then
-    echo "Activating venv"
+    echo "create venv"
     cd ${SD04_DIR}/webui
     python -m venv venv
+fi
+
+# install dependencies
     cd ${SD04_DIR}/webui
     source venv/bin/activate
     pip install --upgrade pip
-    pip install onnxruntime-gpu
+    pip install coloredlogs flatbuffers numpy packaging protobuf==3.20.3 sympy
+    pip install onnxruntime-gpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
     pip install insightface
-    pip install protobuf==3.20.3
+    pip install basicsr
+    pip install sqlalchemy
     deactivate
+
+#copy default parameters if absent
+if [ ! -f "$SD04_DIR/parameters.txt" ]; then
+    cp -v "${SD_INSTALL_DIR}/parameters/04.txt" "$SD04_DIR/parameters.txt"
 fi
 
 # Merge Models, vae, lora, hypernetworks, and outputs
@@ -84,10 +90,8 @@ sl_folder ${SD04_DIR}/webui/models ControlNet ${BASE_DIR}/models controlnet
 sl_folder ${SD04_DIR}/webui outputs ${BASE_DIR}/outputs 04-SD-Next
 
 cd ${SD04_DIR}/webui/
-source venv/bin/activate
-export PATH="/config/04-SD-Next/env/lib/python3.11/site-packages/onnxruntime/capi:$PATH"
 
-pip install typing-extensions==4.8.0 numpy==1.24.4 huggingface_hub==0.18.0 sqlalchemy --upgrade
+#Launch WebUI
 CMD="bash webui.sh"
 while IFS= read -r param; do
     if [[ $param != \#* ]]; then