adds controlnet reference for probably the wrong reasons

zero01101 · Jul 15, 2023 · 6461e3b · 6461e3b
1 parent a83f729
commit 6461e3b
Show file tree

Hide file tree

Showing 5 changed files with 175 additions and 40 deletions.
diff --git a/README.md b/README.md
@@ -16,14 +16,16 @@ this is a completely vanilla javascript and html canvas outpainting convenience
 
 ## features
 
+- SDXL "support"! (please check outpaint/inpaint fill types in the context menus and fiddle with denoising a LOT for img2img, it's touchy)
 - [now available as an extension for webUI!](https://github.com/zero01101/openOutpaint-webUI-extension) you can find it under the default "available" section in the webUI _extensions_ tab
   - **_NOTE: extension still requires `--api` flag in webui-user launch script_**
 - intuitive, convenient outpainting - that's like the whole point right
 - queueable, cancelable dreams - just start a'clickin' all over the place
 - arbitrary dream reticle size - draw the rectangle of your dreams
 - an [effectively infinite](https://github.com/zero01101/openOutpaint/pull/108), resizable, scalable canvas for you to paint all over
   - **_NOTE: v0.0.10 introduces a new "camera control" modifier key - hold [`CTRL`] and use the scrollwheel to zoom (scroll the wheel or use the two-finger vertical gesture on, uh, modern touchpads) and pan (hold the scrollwheel button, or if you don't have one, left-click button) around the canvas_**
-- extremely limited, janky support for a shockingly restrictive list of A1111 extensions including controlnet inpainting for legitimately [magic](https://github.com/Mikubill/sd-webui-controlnet/discussions/1464) [promptless inpainting](https://github.com/Mikubill/sd-webui-controlnet/discussions/1143) and [outpainting](https://github.com/Mikubill/sd-webui-controlnet/discussions/1597)
+- extremely limited, janky support for a shockingly restrictive list of A1111 extensions including controlnet inpainting for legitimately [magic](https://github.com/Mikubill/sd-webui-controlnet/discussions/1464) [promptless inpainting](https://github.com/Mikubill/sd-webui-controlnet/discussions/1143) and [outpainting](https://github.com/Mikubill/sd-webui-controlnet/discussions/1597) and in-line reference preprocessors for keeping existing style while replacing things (reference requires at least 2 controlnet units enabled in A1111 settings), as well as a very very basic dynamic-prompts-on-or-off toggle
+  - **_NOTE: this is_ JANKY, _pull requests greatly welcomed lol_**
 - a very nicely functional and familiar layer system
 - save, load, import, and export workspaces - includes all your layers, history, canvas size, you name it!
 - inpainting/touchup mask brush

diff --git a/index.html b/index.html
@@ -1,4 +1,4 @@
-<!doctype html>
+<!DOCTYPE html>
 <html lang="en-US">
 	<head>
 		<meta charset="utf-8" />
@@ -213,31 +213,59 @@
 						onchange="changeControlNetExtension()"
 						disabled="disabled" />
 					<label for="cbxControlNet">ControlNet In/Outpainting</label>
-					<br class="controlnetElement" />
-					<label id="cnModuleLabel" class="controlnetElement">
-						Preprocessor
+					<br class="controlNetElement" />
+					<label id="cnModuleLabel" class="controlNetElement">
+						Inpaint Preprocessor
 					</label>
-					<div id="controlNetModule-ac-select" class="controlnetElement"></div>
-					<label id="cnModelLabel" class="controlnetElement">Model</label>
-					<div id="controlNetModel-ac-select" class="controlnetElement"></div>
-					<label id="cnControlLabel" class="controlnetElement">
-						Control Mode
+					<div id="controlNetModule-ac-select" class="controlNetElement"></div>
+					<label id="cnModelLabel" class="controlNetElement">Model</label>
+					<div id="controlNetModel-ac-select" class="controlNetElement"></div>
+					<label id="cnControlLabel" class="controlNetElement">
+						Inpaint Mode
 					</label>
-					<select id="controlNetMode-select" class="controlnetElement">
+					<select id="controlNetMode-select" class="controlNetElement">
 						<option value="Balanced">balanced</option>
 						<option value="My prompt is more important">+prompt</option>
 						<option value="ControlNet is more important">+CN</option>
 					</select>
-					<br />
-					<label id="cnResizeLabel" class="controlnetElement">
+					<br class="controlNetElement" />
+					<label id="cnResizeLabel" class="controlNetElement">
 						Resize Mode
 					</label>
-					<select id="controlNetResize-select" class="controlnetElement">
+					<select id="controlNetResize-select" class="controlNetElement">
 						<option value="Just Resize">resize</option>
 						<option value="Crop and Resize">+crop</option>
 						<option value="Resize and Fill">+fill</option>
 					</select>
-					<!-- <div id="referenceStyleFidelity" class="controlnetElement"></div> -->
+					<br class="controlNetElement" />
+					<input
+						type="checkbox"
+						id="cbxControlNetReferenceLayer"
+						onchange="changeControlNetReference()"
+						class="controlNetElement" />
+					<label for="cbxControlNetReferenceLayer" class="controlNetElement">
+						Reference as 1st CN Unit
+					</label>
+					<br class="controlNetReferenceElement" />
+					<label id="cnReferenceModuleLabel" class="controlNetReferenceElement">
+						Reference Preprocessor
+					</label>
+					<div
+						id="controlNetReferenceModule-ac-select"
+						class="controlNetReferenceElement"></div>
+					<div
+						id="controlNetReferenceFidelity"
+						class="controlNetReferenceElement"></div>
+					<label id="cnResizeLabel" class="controlNetReferenceElement">
+						Reference Mode
+					</label>
+					<select
+						id="controlNetReferenceMode-select"
+						class="controlNetReferenceElement">
+						<option value="Balanced">balanced</option>
+						<option value="My prompt is more important">+prompt</option>
+						<option value="ControlNet is more important">+CN</option>
+					</select>
 				</div>
 				<!-- Save/load image section -->
 				<button type="button" class="collapsible">Save/Upscaling</button>
@@ -287,8 +315,8 @@
 					<br />
 					<span id="version">
 						<a href="https://github.com/zero01101/openOutpaint" target="_blank">
-							<s>Alpha release v0.0.16.2</s>
-							v20230715.001
+							<s>Alpha release v0.0.16.3</s>
+							v20230715.002
 						</a>
 						<br />
 						<a
@@ -485,7 +513,7 @@
 		<!-- Basics -->
 		<script src="js/global.js?v=ac30d16" type="text/javascript"></script>
 		<script src="js/defaults.js?v=5b06818" type="text/javascript"></script>
-		<script src="js/extensions.js?v=abb882a" type="text/javascript"></script>
+		<script src="js/extensions.js?v=1fca0fc" type="text/javascript"></script>
 
 		<!-- Base Libs -->
 		<script src="js/lib/util.js?v=379aef7" type="text/javascript"></script>
@@ -511,7 +539,7 @@
 
 		<!-- Content -->
 		<script src="js/prompt.js?v=7a1c68c" type="text/javascript"></script>
-		<script src="js/index.js?v=31d52e7" type="text/javascript"></script>
+		<script src="js/index.js?v=7debf82" type="text/javascript"></script>
 
 		<script
 			src="js/ui/floating/history.js?v=4f29db4"
@@ -525,7 +553,7 @@
 			src="js/ui/tool/generic.js?v=3e678e0"
 			type="text/javascript"></script>
 
-		<script src="js/ui/tool/dream.js?v=ac1643d" type="text/javascript"></script>
+		<script src="js/ui/tool/dream.js?v=06e0164" type="text/javascript"></script>
 		<script
 			src="js/ui/tool/maskbrush.js?v=e9bd0eb"
 			type="text/javascript"></script>

diff --git a/js/extensions.js b/js/extensions.js
@@ -7,8 +7,12 @@ const extensions = {
 	alwaysOnScripts: false,
 	controlNetEnabled: false,
 	controlNetActive: false,
+	controlNetReferenceActive: false,
+	controlNetReferenceFidelity: 0.5,
 	selectedControlNetModel: null,
 	selectedControlNetModule: null,
+	selectedCNReferenceModule: null,
+	controlNetModelCount: 0,
 	dynamicPromptsEnabled: false,
 	dynamicPromptsActive: false,
 	dynamicPromptsAlwaysonScriptName: null, //GRUMBLE GRUMBLE
@@ -18,7 +22,8 @@ const extensions = {
 
 	async getExtensions(
 		controlNetModelAutoComplete,
-		controlNetModuleAutoComplete
+		controlNetModuleAutoComplete,
+		controlNetReferenceModuleAutoComplete
 	) {
 		const allowedExtensions = [
 			"controlnet",
@@ -51,7 +56,8 @@ const extensions = {
 		this.checkForDynamicPrompts();
 		this.checkForControlNet(
 			controlNetModelAutoComplete,
-			controlNetModuleAutoComplete
+			controlNetModuleAutoComplete,
+			controlNetReferenceModuleAutoComplete
 		);
 		//checkForSAM(); //or inpaintAnything or something i dunno
 		//checkForADetailer(); //? this one seems iffy
@@ -78,7 +84,8 @@ const extensions = {
 
 	async checkForControlNet(
 		controlNetModelAutoComplete,
-		controlNetModuleAutoComplete
+		controlNetModuleAutoComplete,
+		controlNetReferenceModuleAutoComplete
 	) {
 		var url = document.getElementById("host").value + "/controlnet/version";
 
@@ -97,8 +104,23 @@ const extensions = {
 				document.getElementById("cbxControlNet").disabled = false;
 				// ok cool so now we can get the models and modules
 				this.getModels(controlNetModelAutoComplete);
-				this.getModules(controlNetModuleAutoComplete);
+				this.getModules(
+					controlNetModuleAutoComplete,
+					controlNetReferenceModuleAutoComplete
+				);
 			}
+			url = document.getElementById("host").value + "/controlnet/settings";
+			try {
+				const response2 = await fetch(url);
+				const data2 = await response2.json();
+				if (data2.control_net_max_models_num < 2) {
+					document.getElementById("cbxControlNetReferenceLayer").disabled =
+						"disabled";
+					console.warn(
+						"[extensions] ControlNet reference layer disabled due to insufficient units enabled in settings - cannot be enabled via API, please increase to at least 2 units manually"
+					);
+				}
+			} catch (ex) {}
 		} catch (e) {
 			// ??
 			global.controlnetAPI = false;
@@ -128,7 +150,10 @@ const extensions = {
 
 		controlNetModelAutoComplete.options = opt;
 	},
-	async getModules(controlNetModuleAutoComplete) {
+	async getModules(
+		controlNetModuleAutoComplete,
+		controlNetReferenceModuleAutoComplete
+	) {
 		const allowedModules = ["reference", "inpaint"];
 		var url = document.getElementById("host").value + "/controlnet/module_list";
 
@@ -156,5 +181,14 @@ const extensions = {
 		});
 
 		controlNetModuleAutoComplete.options = opt;
+
+		opt = this.controlNetModules.module_list
+			.filter((m) => m.includes("reference"))
+			.map((option) => ({
+				name: option,
+				value: option,
+			}));
+
+		controlNetReferenceModuleAutoComplete.options = opt;
 	},
 };
diff --git a/js/index.js b/js/index.js
@@ -172,6 +172,7 @@ function startup() {
 	changeRestoreFaces();
 	changeSyncCursorSize();
 	changeControlNetExtension();
+	changeControlNetReference();
 	checkFocus();
 	refreshScripts();
 }
@@ -424,7 +425,8 @@ async function testHostConnection() {
 							getModels();
 							extensions.getExtensions(
 								controlNetModelAutoComplete,
-								controlNetModuleAutoComplete
+								controlNetModuleAutoComplete,
+								controlNetReferenceModuleAutoComplete
 							);
 							getLoras();
 							// getTIEmbeddings();
@@ -680,7 +682,7 @@ const hrFixUpscalerAutoComplete = createAutoComplete(
 );
 
 let controlNetModelAutoComplete = createAutoComplete(
-	"ControlNet Model",
+	"Inpaint Model",
 	document.getElementById("controlNetModel-ac-select")
 );
 
@@ -689,14 +691,23 @@ controlNetModelAutoComplete.onchange.on(({value}) => {
 });
 
 let controlNetModuleAutoComplete = createAutoComplete(
-	"ControlNet Module",
+	"Inpaint Preprocessor",
 	document.getElementById("controlNetModule-ac-select")
 );
 
 controlNetModuleAutoComplete.onchange.on(({value}) => {
 	extensions.selectedControlNetModule = value;
 });
 
+let controlNetReferenceModuleAutoComplete = createAutoComplete(
+	"Reference Preprocessor",
+	document.getElementById("controlNetReferenceModule-ac-select")
+);
+
+controlNetReferenceModuleAutoComplete.onchange.on(({value}) => {
+	extensions.selectedCNReferenceModule = value;
+});
+
 // const extensionsAutoComplete = createAutoComplete(
 // 	"Extension",
 // 	document.getElementById("extension-ac-select")
@@ -724,6 +735,25 @@ const resSlider = makeSlider(
 			toolbar.currentTool.redraw();
 	}
 );
+
+const refSlider = makeSlider(
+	"Reference Fidelity",
+	document.getElementById("controlNetReferenceFidelity"),
+	"cn_reference_fidelity",
+	0.0,
+	1.0,
+	0.1,
+	0.5,
+	0.01,
+	(v) => {
+		extensions.controlNetReferenceFidelity = v;
+
+		toolbar.currentTool &&
+			toolbar.currentTool.redraw &&
+			toolbar.currentTool.redraw();
+	}
+);
+
 makeSlider(
 	"CFG Scale",
 	document.getElementById("cfgScale"),
@@ -851,11 +881,26 @@ function changeControlNetExtension() {
 		document.getElementById("cbxControlNet").checked;
 	if (extensions.controlNetActive) {
 		document
-			.querySelectorAll(".controlnetElement")
+			.querySelectorAll(".controlNetElement")
+			.forEach((el) => el.classList.remove("invisible"));
+	} else {
+		document
+			.querySelectorAll(".controlNetElement")
+			.forEach((el) => el.classList.add("invisible"));
+	}
+}
+
+function changeControlNetReference() {
+	extensions.controlNetReferenceActive = document.getElementById(
+		"cbxControlNetReferenceLayer"
+	).checked;
+	if (extensions.controlNetReferenceActive) {
+		document
+			.querySelectorAll(".controlNetReferenceElement")
 			.forEach((el) => el.classList.remove("invisible"));
 	} else {
 		document
-			.querySelectorAll(".controlnetElement")
+			.querySelectorAll(".controlNetReferenceElement")
 			.forEach((el) => el.classList.add("invisible"));
 	}
 }