Feature Extractor Example UI

OnnxStack.Console/Examples/UpscaleStreamExample.cs

@@ -15,7 +15,7 @@ public UpscaleStreamExample()
 
         public int Index => 10;
 
-        public string Name => "Streaming Video Upscale Demo";
+        public string Name => "Upscale Video Streaming Demo";
 
         public string Description => "Upscales a video stream";
 

OnnxStack.FeatureExtractor/Pipelines/FeatureExtractorPipeline.cs

@@ -41,6 +41,16 @@ public FeatureExtractorPipeline(string name, FeatureExtractorModel featureExtrac
         public string Name => _name;
 
 
+        /// <summary>
+        /// Loads the model.
+        /// </summary>
+        /// <returns></returns>
+        public Task LoadAsync()
+        {
+            return _featureExtractorModel.LoadAsync();
+        }
+
+
         /// <summary>
         /// Unloads the models.
         /// </summary>

OnnxStack.UI/App.xaml.cs

@@ -47,7 +47,7 @@ public App()
             builder.Services.AddSingleton<IDeviceService, DeviceService>();
             builder.Services.AddSingleton<IStableDiffusionService, StableDiffusionService>();
             builder.Services.AddSingleton<IUpscaleService, UpscaleService>();
-
+            builder.Services.AddSingleton<IFeatureExtractorService, FeatureExtractorService>();
 
             // Build App
             _applicationHost = builder.Build();

OnnxStack.UI/MainWindow.xaml

@@ -87,6 +87,19 @@
                 <views:UpscaleView UISettings="{Binding UISettings}" Margin="0,6,0,0"/>
             </TabItem>
 
+            <!--Feature Extractor-->
+            <TabItem>
+                <TabItem.Header>
+                    <StackPanel Orientation="Horizontal" Margin="5">
+                        <StackPanel Orientation="Horizontal">
+                            <userControls:FontAwesome Icon="&#xf87c;" IconStyle="Light"/>
+                        </StackPanel>
+                        <TextBlock Text="Feature Extractor" Margin="5,0,0,0"/>
+                    </StackPanel>
+                </TabItem.Header>
+                <views:FeatureExtractorView UISettings="{Binding UISettings}" Margin="0,6,0,0"/>
+            </TabItem>
+
             <!--Log Window-->
             <TabItem DockPanel.Dock="Right" HorizontalAlignment="Right">
                 <TabItem.Header>

OnnxStack.UI/MainWindow.xaml.cs

@@ -27,6 +27,7 @@ public MainWindow(OnnxStackUIConfig uiSettings, ILogger<MainWindow> logger)
             UISettings = uiSettings;
             SaveImageCommand = new AsyncRelayCommand<UpscaleResult>(SaveUpscaleImageFile);
             SaveImageResultCommand = new AsyncRelayCommand<ImageResult>(SaveImageResultFile);
+            SaveFeatureImageCommand = new AsyncRelayCommand<FeatureExtractorResult>(SaveFeatureImageFile);
             SaveBlueprintCommand = new AsyncRelayCommand<ImageResult>(SaveBlueprintFile);
             NavigateTextToImageCommand = new AsyncRelayCommand<ImageResult>(NavigateTextToImage);
             NavigateImageToImageCommand = new AsyncRelayCommand<ImageResult>(NavigateImageToImage);
@@ -39,14 +40,13 @@ public MainWindow(OnnxStackUIConfig uiSettings, ILogger<MainWindow> logger)
             InitializeComponent();
         }
 
-
-
         public AsyncRelayCommand WindowMinimizeCommand { get; }
         public AsyncRelayCommand WindowRestoreCommand { get; }
         public AsyncRelayCommand WindowMaximizeCommand { get; }
         public AsyncRelayCommand WindowCloseCommand { get; }
         public AsyncRelayCommand<UpscaleResult> SaveImageCommand { get; }
         public AsyncRelayCommand<ImageResult> SaveImageResultCommand { get; }
+        public AsyncRelayCommand<FeatureExtractorResult> SaveFeatureImageCommand { get; }
         public AsyncRelayCommand<ImageResult> SaveBlueprintCommand { get; }
         public AsyncRelayCommand<ImageResult> NavigateTextToImageCommand { get; }
         public AsyncRelayCommand<ImageResult> NavigateImageToImageCommand { get; }
@@ -151,6 +151,26 @@ private async Task SaveUpscaleImageFile(UpscaleResult imageResult)
         }
 
 
+        private async Task SaveFeatureImageFile(FeatureExtractorResult imageResult)
+        {
+            try
+            {
+                var filename = GetSaveFilename($"feature-{Random.Shared.Next()}");
+                if (string.IsNullOrEmpty(filename))
+                    return;
+
+                var result = await imageResult.Image.SaveImageFileAsync(filename);
+                if (!result)
+                    _logger.LogError("Error saving image");
+
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error saving image");
+            }
+        }
+
+
         private async Task SaveBlueprintFile(ImageResult imageResult)
         {
             try

OnnxStack.UI/Models/FeatureExtractorResult.cs

@@ -0,0 +1,6 @@
+using System.Windows.Media.Imaging;
+
+namespace OnnxStack.UI.Models
+{
+    public record FeatureExtractorResult(BitmapSource Image, double Elapsed);
+}

OnnxStack.UI/Services/FeatureExtractorService.cs

@@ -0,0 +1,119 @@
+using Microsoft.Extensions.Logging;
+using OnnxStack.Core.Config;
+using OnnxStack.Core.Image;
+using OnnxStack.Core.Video;
+using OnnxStack.FeatureExtractor.Common;
+using OnnxStack.FeatureExtractor.Pipelines;
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace OnnxStack.UI.Services
+{
+    public class FeatureExtractorService : IFeatureExtractorService
+    {
+        private readonly ILogger<FeatureExtractorService> _logger;
+        private readonly Dictionary<IOnnxModel, FeatureExtractorPipeline> _pipelines;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FeatureExtractorService"/> class.
+        /// </summary>
+        /// <param name="configuration">The configuration.</param>
+        /// <param name="modelService">The model service.</param>
+        /// <param name="imageService">The image service.</param>
+        public FeatureExtractorService(ILogger<FeatureExtractorService> logger)
+        {
+            _logger = logger;
+            _pipelines = new Dictionary<IOnnxModel, FeatureExtractorPipeline>();
+        }
+
+
+        /// <summary>
+        /// Loads the model.
+        /// </summary>
+        /// <param name="model">The model.</param>
+        /// <returns></returns>
+        public async Task<bool> LoadModelAsync(FeatureExtractorModelSet model)
+        {
+            if (_pipelines.ContainsKey(model))
+                return true;
+
+            var pipeline = CreatePipeline(model);
+            await pipeline.LoadAsync();
+            return _pipelines.TryAdd(model, pipeline);
+        }
+
+
+        /// <summary>
+        /// Unloads the model.
+        /// </summary>
+        /// <param name="model">The model.</param>
+        /// <returns></returns>
+        public async Task<bool> UnloadModelAsync(FeatureExtractorModelSet model)
+        {
+            if (_pipelines.Remove(model, out var pipeline))
+            {
+                await pipeline?.UnloadAsync();
+            }
+            return true;
+        }
+
+
+        /// <summary>
+        /// Determines whether [is model loaded] [the specified model options].
+        /// </summary>
+        /// <param name="model">The model.</param>
+        /// <returns>
+        ///   <c>true</c> if [is model loaded] [the specified model options]; otherwise, <c>false</c>.
+        /// </returns>
+        /// <exception cref="System.NotImplementedException"></exception>
+        public bool IsModelLoaded(FeatureExtractorModelSet model)
+        {
+            return _pipelines.ContainsKey(model);
+        }
+
+
+        /// <summary>
+        /// Generates the feature image.
+        /// </summary>
+        /// <param name="model">The model.</param>
+        /// <param name="inputImage">The input image.</param>
+        /// <returns></returns>
+        public async Task<OnnxImage> GenerateAsync(FeatureExtractorModelSet model, OnnxImage inputImage, CancellationToken cancellationToken = default)
+        {
+            if (!_pipelines.TryGetValue(model, out var pipeline))
+                throw new Exception("Pipeline not found or is unsupported");
+
+            return await pipeline.RunAsync(inputImage, cancellationToken);
+        }
+
+
+        /// <summary>
+        /// Generates the feature video.
+        /// </summary>
+        /// <param name="model">The model.</param>
+        /// <param name="inputVideo">The input video.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        /// <exception cref="System.Exception">Pipeline not found or is unsupported</exception>
+        public async Task<OnnxVideo> GenerateAsync(FeatureExtractorModelSet model, OnnxVideo inputVideo, CancellationToken cancellationToken = default)
+        {
+            if (!_pipelines.TryGetValue(model, out var pipeline))
+                throw new Exception("Pipeline not found or is unsupported");
+
+            return await pipeline.RunAsync(inputVideo, cancellationToken);
+        }
+
+
+        /// <summary>
+        /// Creates the pipeline.
+        /// </summary>
+        /// <param name="modelSet">The model.</param>
+        /// <returns></returns>
+        private FeatureExtractorPipeline CreatePipeline(FeatureExtractorModelSet model)
+        {
+            return FeatureExtractorPipeline.CreatePipeline(model, _logger);
+        }
+    }
+}

OnnxStack.UI/Services/IFeatureExtractorService.cs

@@ -0,0 +1,51 @@
+using OnnxStack.Core.Image;
+using OnnxStack.Core.Video;
+using OnnxStack.FeatureExtractor.Common;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace OnnxStack.UI.Services
+{
+    public interface IFeatureExtractorService
+    {
+
+        /// <summary>
+        /// Loads the model.
+        /// </summary>
+        /// <param name="model">The model.</param>
+        /// <returns></returns>
+        Task<bool> LoadModelAsync(FeatureExtractorModelSet model);
+
+        /// <summary>
+        /// Unloads the model.
+        /// </summary>
+        /// <param name="model">The model.</param>
+        /// <returns></returns>
+        Task<bool> UnloadModelAsync(FeatureExtractorModelSet model);
+
+        /// <summary>
+        /// Determines whether [is model loaded] [the specified model options].
+        /// </summary>
+        /// <param name="model">The modelset.</param>
+        /// <returns>
+        ///   <c>true</c> if [is model loaded] [the specified model options]; otherwise, <c>false</c>.
+        /// </returns>
+        bool IsModelLoaded(FeatureExtractorModelSet model);
+
+        /// <summary>
+        /// Generates the feature image.
+        /// </summary>
+        /// <param name="model">The model.</param>
+        /// <param name="inputImage">The input image.</param>
+        /// <returns></returns>
+        Task<OnnxImage> GenerateAsync(FeatureExtractorModelSet model, OnnxImage inputImage, CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Generates the feature video.
+        /// </summary>
+        /// <param name="model">The model.</param>
+        /// <param name="inputVideo">The input video.</param>
+        /// <returns></returns>
+        Task<OnnxVideo> GenerateAsync(FeatureExtractorModelSet model, OnnxVideo inputVideo, CancellationToken cancellationToken = default);
+    }
+}

OnnxStack.UI/Services/IUpscaleService.cs

@@ -1,10 +1,6 @@
-using Microsoft.ML.OnnxRuntime.Tensors;
-using OnnxStack.Core.Image;
+using OnnxStack.Core.Image;
 using OnnxStack.Core.Video;
 using OnnxStack.ImageUpscaler.Common;
-using SixLabors.ImageSharp;
-using SixLabors.ImageSharp.PixelFormats;
-using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -30,18 +26,26 @@ public interface IUpscaleService
         /// <summary>
         /// Determines whether [is model loaded] [the specified model options].
         /// </summary>
-        /// <param name="modelOptions">The model options.</param>
+        /// <param name="model">The model.</param>
         /// <returns>
         ///   <c>true</c> if [is model loaded] [the specified model options]; otherwise, <c>false</c>.
         /// </returns>
-        bool IsModelLoaded(UpscaleModelSet modelOptions);
+        bool IsModelLoaded(UpscaleModelSet model);
 
         /// <summary>
         /// Generates the upscaled image.
         /// </summary>
-        /// <param name="modelOptions">The model options.</param>
+        /// <param name="model">The model.</param>
+        /// <param name="inputImage">The input image.</param>
+        /// <returns></returns>
+        Task<OnnxImage> GenerateAsync(UpscaleModelSet model, OnnxImage inputImage, CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Generates the upscaled image.
+        /// </summary>
+        /// <param name="model">The model.</param>
         /// <param name="inputImage">The input image.</param>
         /// <returns></returns>
-        Task<OnnxImage> GenerateAsync(UpscaleModelSet modelOptions, OnnxImage inputImage, CancellationToken cancellationToken = default);
+        Task<OnnxVideo> GenerateAsync(UpscaleModelSet model, OnnxVideo inputVideo, CancellationToken cancellationToken = default);
     }
 }

OnnxStack.UI/Services/UpscaleService.cs

@@ -63,26 +63,26 @@ public async Task<bool> UnloadModelAsync(UpscaleModelSet model)
         /// <summary>
         /// Determines whether [is model loaded] [the specified model options].
         /// </summary>
-        /// <param name="modelOptions">The model options.</param>
+        /// <param name="model">The model.</param>
         /// <returns>
         ///   <c>true</c> if [is model loaded] [the specified model options]; otherwise, <c>false</c>.
         /// </returns>
         /// <exception cref="System.NotImplementedException"></exception>
-        public bool IsModelLoaded(UpscaleModelSet modelOptions)
+        public bool IsModelLoaded(UpscaleModelSet model)
         {
-            return _pipelines.ContainsKey(modelOptions);
+            return _pipelines.ContainsKey(model);
         }
 
 
         /// <summary>
         /// Generates the upscaled image.
         /// </summary>
-        /// <param name="modelSet">The model options.</param>
+        /// <param name="model">The model.</param>
         /// <param name="inputImage">The input image.</param>
         /// <returns></returns>
-        public async Task<OnnxImage> GenerateAsync(UpscaleModelSet modelSet, OnnxImage inputImage, CancellationToken cancellationToken = default)
+        public async Task<OnnxImage> GenerateAsync(UpscaleModelSet model, OnnxImage inputImage, CancellationToken cancellationToken = default)
         {
-            if (!_pipelines.TryGetValue(modelSet, out var pipeline))
+            if (!_pipelines.TryGetValue(model, out var pipeline))
                 throw new Exception("Pipeline not found or is unsupported");
 
             return await pipeline.RunAsync(inputImage, cancellationToken);
@@ -92,14 +92,14 @@ public async Task<OnnxImage> GenerateAsync(UpscaleModelSet modelSet, OnnxImage i
         /// <summary>
         /// Generates the upscaled video.
         /// </summary>
-        /// <param name="modelSet">The model set.</param>
+        /// <param name="model">The model.</param>
         /// <param name="inputVideo">The input video.</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns></returns>
         /// <exception cref="System.Exception">Pipeline not found or is unsupported</exception>
-        public async Task<OnnxVideo> GenerateAsync(UpscaleModelSet modelSet, OnnxVideo inputVideo, CancellationToken cancellationToken = default)
+        public async Task<OnnxVideo> GenerateAsync(UpscaleModelSet model, OnnxVideo inputVideo, CancellationToken cancellationToken = default)
         {
-            if (!_pipelines.TryGetValue(modelSet, out var pipeline))
+            if (!_pipelines.TryGetValue(model, out var pipeline))
                 throw new Exception("Pipeline not found or is unsupported");
 
             return await pipeline.RunAsync(inputVideo, cancellationToken);
@@ -109,11 +109,11 @@ public async Task<OnnxVideo> GenerateAsync(UpscaleModelSet modelSet, OnnxVideo i
         /// <summary>
         /// Creates the pipeline.
         /// </summary>
-        /// <param name="modelSet">The model set.</param>
+        /// <param name="model">The model.</param>
         /// <returns></returns>
-        private ImageUpscalePipeline CreatePipeline(UpscaleModelSet modelSet)
+        private ImageUpscalePipeline CreatePipeline(UpscaleModelSet model)
         {
-            return ImageUpscalePipeline.CreatePipeline(modelSet, _logger);
+            return ImageUpscalePipeline.CreatePipeline(model, _logger);
         }
     }
 }