E2E Sample - Live Stream Video Object Detection with Onnx in WPF Desk…

…top App (#602) * Added UWP project with simple camera support * added simple rectangle overlay drawing * Added ML.NET predictions with yolo model * added TODO comment for Bitmap type issue * added shared and wpf project (and it's broken) * reference shared classes from web app * DeepLearning_ObjectDetection_Onnx -> ObjectDetection_Onnx * draw a square * Rough wpf web cam code * added toolkit * view live webcam stream * initial working prototype * use same colors as web app * only provide the model once and share across projs * Renaming * Use MicrosoftMLVersion in csproj files * fix build * one more 1.3.1 -> MicrosoftMLVersion * cleanup performance and comments * use .net core 3.0 * fix build def * fix camera bug on some devices (thanks @eerhardt) * update web project to .net core 3.0 * edits and cleanup to original README * update README to include information on the WPF desktop app * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/README.md Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/README.md Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/README.md Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/README.md Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/README.md Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/README.md Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/README.md Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/README.md Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/README.md Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/OnnxObjectDetectionApp/MainWindow.xaml.cs Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * Update samples/csharp/end-to-end-apps/ObjectDetection-Onnx/OnnxObjectDetectionApp/MainWindow.xaml.cs Co-Authored-By: Brigit Murtaugh <brigit.murtaugh@microsoft.com> * readme edits and formatting. thanks @bamurtaugh! * readme edits and fixes per @nicolehaugen review. thanks!
dotnet · Aug 20, 2019 · 7e8ce49 · 7e8ce49
1 parent bd8bac4
commit 7e8ce49
Show file tree

Hide file tree

Showing 98 changed files with 669 additions and 265 deletions.
diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml
@@ -163,10 +163,16 @@ phases:
 - phase: ObjectDetectionE2EAPP
   queue: Hosted VS2017
   steps:
+  - task: UseDotNet@2
+    displayName: 'Use .NET Core 3.0'
+    inputs:
+      version: 3.0.x
+      includePreviewVersions: true
+      installationPath: $(Agent.ToolsDirectory)/dotnet
   - task: DotNetCoreCLI@2
     displayName: Build Object Detection E2E (Onnx Scorer)
     inputs:
-      projects: '.\samples\csharp\end-to-end-apps\DeepLearning_ObjectDetection_Onnx\OnnxObjectDetectionE2EApp.sln'
+      projects: '.\samples\csharp\end-to-end-apps\ObjectDetection-Onnx\OnnxObjectDetection.sln'
 
 - phase: SalesSpikeChangeDetectionE2E
   queue: Hosted VS2017

diff --git a/...rp/end-to-end-apps/DeepLearning_ObjectDetection_Onnx/OnnxObjectDetectionE2EAPP/Program.cs b/...rp/end-to-end-apps/DeepLearning_ObjectDetection_Onnx/OnnxObjectDetectionE2EAPP/Program.cs
diff --git a/...es/csharp/end-to-end-apps/DeepLearning_ObjectDetection_Onnx/OnnxObjectDetectionE2EApp.sln b/...es/csharp/end-to-end-apps/DeepLearning_ObjectDetection_Onnx/OnnxObjectDetectionE2EApp.sln
diff --git a/samples/csharp/end-to-end-apps/DeepLearning_ObjectDetection_Onnx/README.md b/samples/csharp/end-to-end-apps/DeepLearning_ObjectDetection_Onnx/README.md
diff --git a/...jectDetection_Onnx/.ionide/symbolCache.db → ...jectDetection-Onnx/.ionide/symbolCache.db b/...jectDetection_Onnx/.ionide/symbolCache.db → ...jectDetection-Onnx/.ionide/symbolCache.db
diff --git a/samples/csharp/end-to-end-apps/ObjectDetection-Onnx/OnnxObjectDetection.sln b/samples/csharp/end-to-end-apps/ObjectDetection-Onnx/OnnxObjectDetection.sln
@@ -0,0 +1,93 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.28803.452
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OnnxObjectDetectionWeb", "OnnxObjectDetectionWeb\OnnxObjectDetectionWeb.csproj", "{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OnnxObjectDetection", "OnnxObjectDetection\OnnxObjectDetection.csproj", "{7B159949-6D64-41B2-A30F-1952FA8EBA3E}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OnnxObjectDetectionApp", "OnnxObjectDetectionApp\OnnxObjectDetectionApp.csproj", "{30411590-5517-4E40-8AC6-88E916B66B09}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|ARM = Debug|ARM
+		Debug|ARM64 = Debug|ARM64
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|ARM = Release|ARM
+		Release|ARM64 = Release|ARM64
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Debug|ARM.ActiveCfg = Debug|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Debug|ARM.Build.0 = Debug|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Debug|ARM64.ActiveCfg = Debug|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Debug|ARM64.Build.0 = Debug|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Debug|x64.Build.0 = Debug|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Debug|x86.Build.0 = Debug|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Release|ARM.ActiveCfg = Release|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Release|ARM.Build.0 = Release|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Release|ARM64.ActiveCfg = Release|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Release|ARM64.Build.0 = Release|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Release|x64.ActiveCfg = Release|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Release|x64.Build.0 = Release|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Release|x86.ActiveCfg = Release|Any CPU
+		{4A91FD3C-80FC-40E9-9A0B-0F832B313C24}.Release|x86.Build.0 = Release|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Debug|ARM.ActiveCfg = Debug|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Debug|ARM.Build.0 = Debug|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Debug|ARM64.ActiveCfg = Debug|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Debug|ARM64.Build.0 = Debug|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Debug|x64.Build.0 = Debug|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Debug|x86.Build.0 = Debug|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Release|ARM.ActiveCfg = Release|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Release|ARM.Build.0 = Release|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Release|ARM64.ActiveCfg = Release|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Release|ARM64.Build.0 = Release|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Release|x64.ActiveCfg = Release|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Release|x64.Build.0 = Release|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Release|x86.ActiveCfg = Release|Any CPU
+		{7B159949-6D64-41B2-A30F-1952FA8EBA3E}.Release|x86.Build.0 = Release|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Debug|ARM.ActiveCfg = Debug|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Debug|ARM.Build.0 = Debug|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Debug|ARM64.ActiveCfg = Debug|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Debug|ARM64.Build.0 = Debug|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Debug|x64.Build.0 = Debug|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Debug|x86.Build.0 = Debug|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Release|Any CPU.Build.0 = Release|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Release|ARM.ActiveCfg = Release|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Release|ARM.Build.0 = Release|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Release|ARM64.ActiveCfg = Release|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Release|ARM64.Build.0 = Release|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Release|x64.ActiveCfg = Release|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Release|x64.Build.0 = Release|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Release|x86.ActiveCfg = Release|Any CPU
+		{30411590-5517-4E40-8AC6-88E916B66B09}.Release|x86.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {E4E2676A-8816-4A2F-A0F0-1E2718DAFC78}
+	EndGlobalSection
+EndGlobal
diff --git a/...ionE2EAPP/ML/DataModels/ImageInputData.cs → ...Detection/ML/DataModels/ImageInputData.cs b/...ionE2EAPP/ML/DataModels/ImageInputData.cs → ...Detection/ML/DataModels/ImageInputData.cs
@@ -1,7 +1,7 @@
 using Microsoft.ML.Transforms.Image;
 using System.Drawing;
 
-namespace OnnxObjectDetectionE2EAPP
+namespace OnnxObjectDetection
 {
     public class ImageInputData
     {

diff --git a/...PP/ML/DataModels/ImageObjectPrediction.cs → ...on/ML/DataModels/ImageObjectPrediction.cs b/...PP/ML/DataModels/ImageObjectPrediction.cs → ...on/ML/DataModels/ImageObjectPrediction.cs
@@ -1,6 +1,6 @@
 using Microsoft.ML.Data;
 
-namespace OnnxObjectDetectionE2EAPP
+namespace OnnxObjectDetection
 {
     public class ImageObjectPrediction
     {

diff --git a/...nE2EAPP/ML/OnnxModel/TinyYolo2_model.onnx → ...tection/ML/OnnxModel/TinyYolo2_model.onnx b/...nE2EAPP/ML/OnnxModel/TinyYolo2_model.onnx → ...tection/ML/OnnxModel/TinyYolo2_model.onnx
diff --git a/...tectionE2EAPP/ML/OnnxModelConfigurator.cs → ...jectDetection/ML/OnnxModelConfigurator.cs b/...tectionE2EAPP/ML/OnnxModelConfigurator.cs → ...jectDetection/ML/OnnxModelConfigurator.cs
@@ -1,12 +1,9 @@
-using System;
+using Microsoft.ML;
+using Microsoft.ML.Transforms.Image;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
-using Microsoft.ML;
-using Microsoft.ML.Transforms.Image;
-using OnnxObjectDetectionE2EAPP.Utilities;
 
-namespace OnnxObjectDetectionE2EAPP.MLModel
+namespace OnnxObjectDetection
 {
     public class OnnxModelConfigurator
     {
@@ -16,7 +13,8 @@ public class OnnxModelConfigurator
         public OnnxModelConfigurator(string onnxModelFilePath)
         {
             _mlContext = new MLContext();
-            // Model creation and pipeline definition for images needs to run just once, so calling it from the constructor:
+            // Model creation and pipeline definition for images needs to run just once,
+            // so calling it from the constructor:
             _mlModel = SetupMlNetModel(onnxModelFilePath);
         }
 
@@ -28,14 +26,13 @@ public struct ImageSettings
 
         public struct TinyYoloModelSettings
         {
-            // for checking TIny yolo2 Model input and  output  parameter names,
-            //you can use tools like Netron, 
-            // which is installed by Visual Studio AI Tools
+            // To check Tiny Yolo2 Model input and output parameter names,
+            // you can use tools like Netron: https://github.com/lutzroeder/netron
 
-            // input tensor name
+            // Input tensor name
             public const string ModelInput = "image";
 
-            // output tensor name
+            // Output tensor name
             public const string ModelOutput = "grid";
         }
 
@@ -52,11 +49,15 @@ public ITransformer SetupMlNetModel(string onnxModelFilePath)
             return mlNetModel;
         }
 
+        public PredictionEngine<ImageInputData, ImageObjectPrediction> GetMlNetPredictionEngine()
+        {
+            return _mlContext.Model.CreatePredictionEngine<ImageInputData, ImageObjectPrediction>(_mlModel);
+        }
+
         public void SaveMLNetModel(string mlnetModelFilePath)
         {
             // Save/persist the model to a .ZIP file to be loaded by the PredictionEnginePool
             _mlContext.Model.Save(_mlModel, null, mlnetModelFilePath);
         }
     }
 }
-