fix(api): reduce copies, fix function signatures

api/onnx_web/image.py

@@ -13,21 +13,6 @@ def blend_imult(a):
     return 1.0 - blend_mult(a)
 
 
-def blend_mask_source(source: Tuple[int, int, int], mask: Tuple[int, int, int], noise: Tuple[int, int, int]) -> Tuple[int, int, int]:
-    '''
-    Blend operation, linear interpolation from noise to source based on mask: `(s * (1 - m)) + (n * m)`
-    Black = noise
-    White = source
-    '''
-    return (
-        int((source[0] * blend_mult(mask[0])) +
-            (noise[0] * blend_imult(mask[0]))),
-        int((source[1] * blend_mult(mask[1])) +
-            (noise[1] * blend_imult(mask[1]))),
-        int((source[2] * blend_mult(mask[2])) +
-            (noise[2] * blend_imult(mask[2]))),
-    )
-
 def blend_source_mask(source: Tuple[int, int, int], mask: Tuple[int, int, int], noise: Tuple[int, int, int]) -> Tuple[int, int, int]:
     '''
     Blend operation, linear interpolation from source to noise based on mask: `(s * (1 - m)) + (n * m)`
@@ -44,23 +29,29 @@ def blend_source_mask(source: Tuple[int, int, int], mask: Tuple[int, int, int],
     )
 
 
-def mask_filter_gaussian(mask_image: Image, dims: Tuple[int, int], origin: Tuple[int, int], rounds=3) -> Tuple[float, float, float]:
-    '''
-    Gaussian blur, source image centered on white canvas.
-    '''
+def mask_filter_none(mask_image: Image, dims: Tuple[int, int], origin: Tuple[int, int], fill='white') -> Image:
     width, height = dims
 
-    noise = Image.new('RGB', (width, height), 'white')
+    noise = Image.new('RGB', (width, height), fill)
     noise.paste(mask_image, origin)
 
+    return noise
+
+
+def mask_filter_gaussian(mask_image: Image, dims: Tuple[int, int], origin: Tuple[int, int], rounds=3) -> Image:
+    '''
+    Gaussian blur, source image centered on white canvas.
+    '''
+    noise = mask_filter_none(mask_image, dims, origin)
+
     for i in range(rounds):
         blur = noise.filter(ImageFilter.GaussianBlur(5))
         noise = ImageChops.screen(noise, blur)
 
     return noise
 
 
-def noise_source_fill(source_image: Image, dims: Tuple[int, int], origin: Tuple[int, int], fill='white') -> Tuple[float, float, float]:
+def noise_source_fill(source_image: Image, dims: Tuple[int, int], origin: Tuple[int, int], fill='white') -> Image:
     '''
     Identity transform, source image centered on white canvas.
     '''
@@ -72,7 +63,7 @@ def noise_source_fill(source_image: Image, dims: Tuple[int, int], origin: Tuple[
     return noise
 
 
-def noise_source_gaussian(source_image: Image, dims: Tuple[int, int], origin: Tuple[int, int], rounds=3) -> Tuple[float, float, float]:
+def noise_source_gaussian(source_image: Image, dims: Tuple[int, int], origin: Tuple[int, int], rounds=3) -> Image:
     '''
     Gaussian blur, source image centered on white canvas.
     '''
@@ -85,7 +76,7 @@ def noise_source_gaussian(source_image: Image, dims: Tuple[int, int], origin: Tu
     return noise
 
 
-def noise_source_uniform(source_image: Image, dims: Tuple[int, int], origin: Tuple[int, int]) -> Tuple[float, float, float]:
+def noise_source_uniform(source_image: Image, dims: Tuple[int, int], origin: Tuple[int, int]) -> Image:
     width, height = dims
     size = width * height
 
@@ -107,7 +98,7 @@ def noise_source_uniform(source_image: Image, dims: Tuple[int, int], origin: Tup
     return noise
 
 
-def noise_source_normal(source_image: Image, dims: Tuple[int, int], origin: Tuple[int, int]) -> Tuple[float, float, float]:
+def noise_source_normal(source_image: Image, dims: Tuple[int, int], origin: Tuple[int, int]) -> Image:
     width, height = dims
     size = width * height
 
@@ -129,7 +120,7 @@ def noise_source_normal(source_image: Image, dims: Tuple[int, int], origin: Tupl
     return noise
 
 
-def noise_source_histogram(source_image: Image, dims: Tuple[int, int], origin: Tuple[int, int]) -> Tuple[float, float, float]:
+def noise_source_histogram(source_image: Image, dims: Tuple[int, int], origin: Tuple[int, int]) -> Image:
     r, g, b = source_image.split()
     width, height = dims
     size = width * height
@@ -180,9 +171,9 @@ def expand_image(
     full_source = Image.new('RGB', dims, fill)
     full_source.paste(source_image, origin)
 
-    full_mask = Image.new('RGB', dims, fill)
-    full_mask = mask_filter_gaussian(full_mask, dims, origin)
-    full_mask.paste(mask_image, origin)
+    full_mask = mask_filter(mask_image, dims, origin)
+    # full_mask = Image.new('RGB', dims, fill)
+    # full_mask.paste(mask_image, origin)
 
     full_noise = noise_source(source_image, dims, origin)
 

api/onnx_web/serve.py

@@ -29,7 +29,18 @@
 from os import environ, makedirs, path, scandir
 from typing import Any, Dict, Tuple, Union
 
-from .image import expand_image, noise_source_gaussian, noise_source_histogram, noise_source_normal, noise_source_fill, noise_source_uniform, blend_source_mask, blend_imult, blend_mask_source, blend_mult
+from .image import (
+    expand_image,
+    # mask filters
+    mask_filter_gaussian,
+    mask_filter_none,
+    # noise sources
+    noise_source_gaussian,
+    noise_source_histogram,
+    noise_source_normal,
+    noise_source_fill,
+    noise_source_uniform,
+)
 
 import json
 import numpy as np
@@ -79,9 +90,9 @@
     'normal': noise_source_normal,
     'uniform': noise_source_uniform,
 }
-blend_modes = {
-    'mask-source': blend_mask_source,
-    'source-mask': blend_source_mask,
+mask_filters = {
+    'none': mask_filter_none,
+    'gaussian': mask_filter_gaussian,
 }
 
 
@@ -294,7 +305,7 @@ def run_inpaint_pipeline(
     top: int,
     bottom: int,
     noise_source: Any,
-    blend_op: Any
+    mask_filter: Any
 ):
     pipe = load_pipeline(OnnxStableDiffusionInpaintPipeline,
                          model, provider, scheduler)
@@ -309,7 +320,7 @@ def run_inpaint_pipeline(
             mask_image,
             (left, right, top, bottom),
             noise_source=noise_source,
-            blend_op=blend_op)
+            mask_filter=mask_filter)
 
         if environ.get('DEBUG') is not None:
             source_image.save('./last-source.png')
@@ -391,9 +402,9 @@ def introspect():
     }
 
 
-@app.route('/api/settings/blends')
-def list_blend_sources():
-    return jsonify(list(blend_modes.keys()))
+@app.route('/api/settings/masks')
+def list_mask_filters():
+    return jsonify(list(mask_filters.keys()))
 
 
 @app.route('/api/settings/models')
@@ -505,9 +516,9 @@ def inpaint():
     bottom = get_and_clamp_int(
         request.args, 'bottom', 0, config_params.get('height').get('max'), 0)
 
+    mask_filter= get_from_map(request.args, 'filter', mask_filters, 'none')
     noise_source = get_from_map(
         request.args, 'noise', noise_sources, 'histogram')
-    blend_op = get_from_map(request.args, 'blend', blend_modes, 'mask-source')
 
     (output_file, output_full) = make_output_path(
         'inpaint', seed, (prompt, cfg, steps, height, width, seed, left, right, top, bottom))
@@ -536,7 +547,7 @@ def inpaint():
         top,
         bottom,
         noise_source,
-        blend_op)
+        mask_filter)
 
     return jsonify({
         'output': output_file,