Skip to content

Commit

Permalink
Linux Fixes (carson-katri#384)
Browse files Browse the repository at this point in the history 
* isolate install_pip() from user site

* kornia SSLCertVerificationError mitigation

* simplify kornia SSLCertVerificationError mitigation

requests uses certifi.where() and setting SSL_CERT_FILE will allow urllib to use the same certs

* redirect subprocess's stdout to stderr

* remove model preloading from install dependencies operator

* use certifi for whole backend

* fix dependencies missing warning

* fix installing when certain directories are read only

* install pip to .python_dependencies

* add check argument

* fix default argument
  • Loading branch information
@NullSenseStudio @JasonHoku
NullSenseStudio authored and JasonHoku committed Dec 20, 2022
1 parent a0b653d commit 1eaee9f
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 93 deletions.
12 changes: 11 additions & 1 deletion generator_process/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,9 @@ def __init__(self, backend: BackendTarget):
self.intent_backends = {}
self.stdin = sys.stdin.buffer
self.stdout = sys.stdout.buffer
sys.stdout = open(os.devnull, 'w') # prevent stable diffusion logs from breaking ipc
# stdin and stdout are piped for ipc, they should not be accessed through usual methods
# stdout can be redirected to stderr so it's still visible within the parent process's console
sys.stdout = sys.stderr
self.stderr = sys.stderr
self.shared_memory = None
self.stop_requested = False
Expand Down Expand Up @@ -321,6 +323,14 @@ def main():
if args.backend == BackendTarget.LOCAL:
from ldm.invoke import txt2mask
txt2mask.CLIPSEG_WEIGHTS = CLIPSEG_WEIGHTS_PATH

try:
import certifi
os.environ["SSL_CERT_FILE"] = certifi.where()
using_certifi = True
except ModuleNotFoundError:
using_certifi = False
print(f"Using certifi: {using_certifi}")

back.thread.start()
back.main_loop()
Expand Down
13 changes: 0 additions & 13 deletions generator_process/intents/prompt_to_image.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,18 +58,6 @@ def view_step(samples, i):
self.send_action(Action.STEP_NO_SHOW, step=step)

def preload_models():
import urllib
import ssl
urlopen = urllib.request.urlopen
def urlopen_decroator(func):
def urlopen(*args, **kwargs):
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
return func(*args, **kwargs, context=ssl_context)
return urlopen
urllib.request.urlopen = urlopen_decroator(urllib.request.urlopen)

tqdm = None
try:
from huggingface_hub.utils.tqdm import tqdm as hfh_tqdm
Expand Down Expand Up @@ -123,7 +111,6 @@ def update(tqdm_self, n=1):
CLIPDensePredT(version='ViT-B/16', reduce_dim=64)

tqdm.update = old_update
urllib.request.urlopen = urlopen

from transformers.utils.hub import TRANSFORMERS_CACHE
model_paths = {'bert-base-uncased', 'openai--clip-vit-large-patch14'}
Expand Down
134 changes: 106 additions & 28 deletions operators/install_dependencies.py
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
import bpy
import os
import site
import sys
import importlib
import sysconfig
import subprocess
import requests
import tarfile
from enum import IntEnum

from ..absolute_path import absolute_path

def install_pip():
class PipInstall(IntEnum):
DEPENDENCIES = 1
STANDARD = 2
USER_SITE = 3

def install_pip(method = PipInstall.STANDARD):
"""
Installs pip if not already present. Please note that ensurepip.bootstrap() also calls pip, which adds the
environment variable PIP_REQ_TRACKER. After ensurepip.bootstrap() finishes execution, the directory doesn't exist
Expand All @@ -20,35 +26,104 @@ def install_pip():
:return:
"""

try:
# Check if pip is already installed
subprocess.run([sys.executable, "-m", "pip", "--version"], check=True)
except subprocess.CalledProcessError:
import ensurepip
import ensurepip

ensurepip.bootstrap()
if method == PipInstall.DEPENDENCIES:
# ensurepip doesn't have a useful way of installing to a specific directory.
# root parameter can be used, but it just concatenates that to the beginning of
# where it decides to install to, causing a more complicated path to where it installs.
wheels = {}
for name, package in ensurepip._get_packages().items():
if package.wheel_name:
whl = os.path.join(os.path.dirname(ensurepip.__file__), "_bundled", package.wheel_name)
else:
whl = package.wheel_path
wheels[name] = whl
pip_whl = os.path.join(wheels['pip'], 'pip')
subprocess.run([sys.executable, pip_whl, "install", *wheels.values(), "--upgrade", "--no-index", "--no-deps", "--no-cache-dir", "--target", absolute_path(".python_dependencies")], check=True)
return

# STANDARD or USER_SITE
no_user = os.environ.get("PYTHONNOUSERSITE", None)
if method == PipInstall.STANDARD:
os.environ["PYTHONNOUSERSITE"] = "1"
else:
os.environ.pop("PYTHONNOUSERSITE", None)
try:
ensurepip.bootstrap(user=method==PipInstall.USER_SITE)
finally:
os.environ.pop("PIP_REQ_TRACKER", None)
if no_user:
os.environ["PYTHONNOUSERSITE"] = no_user
else:
os.environ.pop("PYTHONNOUSERSITE", None)

def install_pip_any(*methods):
methods = methods or PipInstall
for method in methods:
print(f"Attempting to install pip: {PipInstall(method).name}")
try:
install_pip(method)
return method
except:
import traceback
traceback.print_exc()

def get_pip_install():
def run(pip):
if os.path.exists(pip):
try:
subprocess.run([sys.executable, pip, "--version"], check=True)
return True
except subprocess.CalledProcessError:
pass
return False

if run(absolute_path(".python_dependencies/pip")):
return PipInstall.DEPENDENCIES

# This seems to not raise CalledProcessError while debugging in vscode, but works fine in normal use.
# subprocess.run([sys.executable, "-s", "-m", "pip", "--version"], check=True)
# Best to check if the module directory exists first.
for path in site.getsitepackages():
if run(os.path.join(path,"pip")):
return PipInstall.STANDARD

def install_and_import_requirements(requirements_txt=None):
if run(os.path.join(site.getusersitepackages(),"pip")):
return PipInstall.USER_SITE


def install_and_import_requirements(requirements_txt=None, pip_install=PipInstall.STANDARD):
"""
Installs all modules in the 'requirements.txt' file.
"""
environ_copy = dict(os.environ)
environ_copy["PYTHONNOUSERSITE"] = "1"

python_devel_tgz_path = absolute_path('python-devel.tgz')
response = requests.get(f"https://www.python.org/ftp/python/{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}/Python-{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}.tgz")
open(python_devel_tgz_path, 'wb').write(response.content)
python_devel_tgz = tarfile.open(python_devel_tgz_path)
if pip_install != PipInstall.USER_SITE:
environ_copy["PYTHONNOUSERSITE"] = "1"
if pip_install == PipInstall.DEPENDENCIES:
environ_copy["PYTHONPATH"] = absolute_path(".python_dependencies")
python_include_dir = sysconfig.get_paths()['include']
def members(tf):
prefix = f"Python-{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}/Include/"
l = len(prefix)
for member in tf.getmembers():
if member.path.startswith(prefix):
member.path = member.path[l:]
yield member
python_devel_tgz.extractall(path=python_include_dir, members=members(python_devel_tgz))
if not os.path.exists(python_include_dir):
try:
os.makedirs(python_include_dir)
finally:
pass
if os.access(python_include_dir, os.W_OK):
print("downloading additional include files")
python_devel_tgz_path = absolute_path('python-devel.tgz')
response = requests.get(f"https://www.python.org/ftp/python/{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}/Python-{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}.tgz")
open(python_devel_tgz_path, 'wb').write(response.content)
python_devel_tgz = tarfile.open(python_devel_tgz_path)
def members(tf):
prefix = f"Python-{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}/Include/"
l = len(prefix)
for member in tf.getmembers():
if member.path.startswith(prefix):
member.path = member.path[l:]
yield member
python_devel_tgz.extractall(path=python_include_dir, members=members(python_devel_tgz))
else:
print(f"skipping include files, can't write to {python_include_dir}",file=sys.stderr)

requirements_path = requirements_txt
if requirements_path is None:
Expand All @@ -57,7 +132,7 @@ def members(tf):
else: # Use CUDA dependencies by default on Linux/Windows.
# These are not the submodule dependencies from the `development` branch, but use the `main` branch deps for PyTorch 1.11.0.
requirements_path = 'requirements-win-torch-1-11-0.txt'
subprocess.run([sys.executable, "-m", "pip", "install", "-r", absolute_path(requirements_path), "--no-cache-dir", "--target", absolute_path('.python_dependencies')], check=True, env=environ_copy, cwd=absolute_path("stable_diffusion/"))
subprocess.run([sys.executable, "-m", "pip", "install", "-r", absolute_path(requirements_path), "--upgrade", "--no-cache-dir", "--target", absolute_path('.python_dependencies')], check=True, env=environ_copy, cwd=absolute_path("stable_diffusion/"))

class InstallDependencies(bpy.types.Operator):
bl_idname = "stable_diffusion.install_dependencies"
Expand All @@ -71,12 +146,15 @@ def execute(self, context):
bpy.ops.wm.console_toggle()

try:
install_pip()
install_and_import_requirements(requirements_txt=context.scene.dream_textures_requirements_path)
pip_install = get_pip_install()
if pip_install is None:
pip_install = install_pip_any()
if pip_install is None:
raise ImportError(f'Pip could not be installed. You may have to manually install pip into {absolute_path(".python_dependencies")}')

install_and_import_requirements(requirements_txt=context.scene.dream_textures_requirements_path, pip_install=pip_install)
except (subprocess.CalledProcessError, ImportError) as err:
self.report({"ERROR"}, str(err))
return {"CANCELLED"}

subprocess.run([sys.executable, absolute_path("stable_diffusion/scripts/preload_models.py")], check=True, cwd=absolute_path("stable_diffusion/"))

return {"FINISHED"}
101 changes: 50 additions & 51 deletions preferences.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,56 +104,56 @@ def draw(self, context):

if not weights_installed:
layout.label(text="Complete the following steps to finish setting up the addon:")

if len(os.listdir(absolute_path(".python_dependencies"))) < 2:
missing_sd_box = layout.box()
missing_sd_box.label(text="Dependencies Missing", icon="ERROR")
missing_sd_box.label(text="You've likely downloaded source instead of release by accident.")
missing_sd_box.label(text="Follow the instructions to install for your platform.")
missing_sd_box.operator(OpenLatestVersion.bl_idname, text="Download Latest Release")
return

has_local = len(os.listdir(absolute_path("stable_diffusion"))) > 0
if has_local:
dependencies_box = layout.box()
dependencies_box.label(text="Dependencies Located", icon="CHECKMARK")
dependencies_box.label(text="All dependencies (except for model weights) are included in the release.")

model_weights_box = layout.box()
model_weights_box.label(text="Setup Model Weights", icon="SETTINGS")
if weights_installed:
model_weights_box.label(text="Model weights setup successfully.", icon="CHECKMARK")
else:
model_weights_box.label(text="The model weights are not distributed with the addon.")
model_weights_box.label(text="Follow the steps below to download and install them.")
model_weights_box.label(text="1. Download the file 'sd-v1-4.ckpt'")
model_weights_box.operator(OpenHuggingFace.bl_idname, icon="URL")
model_weights_box.label(text="2. Select the downloaded weights to install.")
model_weights_box.operator(ImportWeights.bl_idname, text="Import Model Weights", icon="IMPORT")
model_weights_box.template_list("UI_UL_list", "dream_textures_weights", self, "weights", self, "active_weights")
model_weights_box.operator(DeleteSelectedWeights.bl_idname, text="Delete Selected Weights", icon="X")

dream_studio_box = layout.box()
dream_studio_box.label(text=f"DreamStudio{' (Optional)' if has_local else ''}", icon="HIDE_OFF")
dream_studio_box.label(text=f"Link to your DreamStudio account to run in the cloud{' instead of locally.' if has_local else '.'}")
key_row = dream_studio_box.row()
key_row.prop(self, "dream_studio_key", text="Key")
key_row.operator(OpenDreamStudio.bl_idname, text="Find Your Key", icon="KEYINGSET")

if weights_installed or len(self.dream_studio_key) > 0:
complete_box = layout.box()
complete_box.label(text="Addon Setup Complete", icon="CHECKMARK")
complete_box.label(text="To locate the interface:")
complete_box.label(text="1. Open an Image Editor or Shader Editor space")
complete_box.label(text="2. Enable 'View' > 'Sidebar'")
complete_box.label(text="3. Select the 'Dream' tab")

if default_presets_missing():
presets_box = layout.box()
presets_box.label(text="Default Presets", icon="PRESET")
presets_box.label(text="It looks like you removed some of the default presets.")
presets_box.label(text="You can restore them here.")
presets_box.operator(RestoreDefaultPresets.bl_idname, icon="RECOVER_LAST")
has_dependencies = len(os.listdir(absolute_path(".python_dependencies"))) > 2
if has_dependencies:
has_local = len(os.listdir(absolute_path("stable_diffusion"))) > 0
if has_local:
dependencies_box = layout.box()
dependencies_box.label(text="Dependencies Located", icon="CHECKMARK")
dependencies_box.label(text="All dependencies (except for model weights) are included in the release.")

model_weights_box = layout.box()
model_weights_box.label(text="Setup Model Weights", icon="SETTINGS")
if weights_installed:
model_weights_box.label(text="Model weights setup successfully.", icon="CHECKMARK")
else:
model_weights_box.label(text="The model weights are not distributed with the addon.")
model_weights_box.label(text="Follow the steps below to download and install them.")
model_weights_box.label(text="1. Download the file 'sd-v1-4.ckpt'")
model_weights_box.operator(OpenHuggingFace.bl_idname, icon="URL")
model_weights_box.label(text="2. Select the downloaded weights to install.")
model_weights_box.operator(ImportWeights.bl_idname, text="Import Model Weights", icon="IMPORT")
model_weights_box.template_list("UI_UL_list", "dream_textures_weights", self, "weights", self, "active_weights")
model_weights_box.operator(DeleteSelectedWeights.bl_idname, text="Delete Selected Weights", icon="X")

dream_studio_box = layout.box()
dream_studio_box.label(text=f"DreamStudio{' (Optional)' if has_local else ''}", icon="HIDE_OFF")
dream_studio_box.label(text=f"Link to your DreamStudio account to run in the cloud{' instead of locally.' if has_local else '.'}")
key_row = dream_studio_box.row()
key_row.prop(self, "dream_studio_key", text="Key")
key_row.operator(OpenDreamStudio.bl_idname, text="Find Your Key", icon="KEYINGSET")

if weights_installed or len(self.dream_studio_key) > 0:
complete_box = layout.box()
complete_box.label(text="Addon Setup Complete", icon="CHECKMARK")
complete_box.label(text="To locate the interface:")
complete_box.label(text="1. Open an Image Editor or Shader Editor space")
complete_box.label(text="2. Enable 'View' > 'Sidebar'")
complete_box.label(text="3. Select the 'Dream' tab")

if default_presets_missing():
presets_box = layout.box()
presets_box.label(text="Default Presets", icon="PRESET")
presets_box.label(text="It looks like you removed some of the default presets.")
presets_box.label(text="You can restore them here.")
presets_box.operator(RestoreDefaultPresets.bl_idname, icon="RECOVER_LAST")
else:
missing_dependencies_box = layout.box()
missing_dependencies_box.label(text="Dependencies Missing", icon="ERROR")
missing_dependencies_box.label(text="You've likely downloaded source instead of release by accident.")
missing_dependencies_box.label(text="Follow the instructions to install for your platform.")
missing_dependencies_box.operator(OpenLatestVersion.bl_idname, text="Download Latest Release")

contributors_box = layout.box()
contributors_box.label(text="Contributors", icon="COMMUNITY")
Expand All @@ -165,8 +165,7 @@ def draw(self, context):
developer_box.label(text="Development Tools", icon="CONSOLE")
developer_box.label(text="This section is for addon development only. You are seeing this because you have 'Developer Extras' enabled.")
developer_box.label(text="Do not use any operators in this section unless you are setting up a development environment.")
already_installed = len(os.listdir(absolute_path(".python_dependencies"))) > 0
if already_installed:
if has_dependencies:
warn_box = developer_box.box()
warn_box.label(text="Dependencies already installed. Only install below if you developing the addon", icon="CHECKMARK")
developer_box.prop(context.scene, 'dream_textures_requirements_path')
Expand Down

0 comments on commit 1eaee9f

Please sign in to comment.