Improve ONNX Object Detection Flow

samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetData.cs

@@ -1,6 +1,9 @@
-using Microsoft.ML.Data;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Microsoft.ML.Data;
 
-namespace ObjectDetection
+namespace ObjectDetection.DataStructures
 {
     public class ImageNetData
     {
@@ -9,11 +12,13 @@ public class ImageNetData
 
         [LoadColumn(1)]
         public string Label;
-    }
 
-    public class ImageNetDataProbability : ImageNetData
-    {
-        public string PredictedLabel;
-        public float Probability { get; set; }
+        public static IEnumerable<ImageNetData> ReadFromFile(string imageFolder)
+        {
+            return Directory
+                .GetFiles(imageFolder)
+                .Where(filePath => Path.GetExtension(filePath) != ".md")
+                .Select(filePath => new ImageNetData { ImagePath = filePath, Label = Path.GetFileName(filePath) });
+        }
     }
 }

samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetPrediction.cs

@@ -1,10 +1,10 @@
 using Microsoft.ML.Data;
 
-namespace ObjectDetection
+namespace ObjectDetection.DataStructures
 {
     public class ImageNetPrediction
     {
-        [ColumnName(OnnxModelScorer.TinyYoloModelSettings.ModelOutput)]
+        [ColumnName("grid")]
         public float[] PredictedLabels;
     }
 }

samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs

@@ -1,10 +1,10 @@
 using System;
 using System.Collections.Generic;
-using System.Drawing;
-using System.Drawing.Drawing2D;
-using System.IO;
 using System.Linq;
 using Microsoft.ML;
+using Microsoft.ML.Data;
+using ObjectDetection.DataStructures;
+using ObjectDetection.YoloParser;
 
 namespace ObjectDetection
 {
@@ -15,13 +15,12 @@ class OnnxModelScorer
         private readonly MLContext mlContext;
 
         private IList<YoloBoundingBox> _boundingBoxes = new List<YoloBoundingBox>();
-        private readonly YoloWinMlParser _parser = new YoloWinMlParser();
 
-        public OnnxModelScorer(string imagesFolder, string modelLocation)
+        public OnnxModelScorer(string imagesFolder, string modelLocation, MLContext mlContext)
         {
             this.imagesFolder = imagesFolder;
             this.modelLocation = modelLocation;
-            mlContext = new MLContext();
+            this.mlContext = mlContext;
         }
 
         public struct ImageNetSettings
@@ -32,7 +31,7 @@ public struct ImageNetSettings
 
         public struct TinyYoloModelSettings
         {
-            // for checking TIny yolo2 Model input and  output  parameter names,
+            // for checking Tiny yolo2 Model input and  output  parameter names,
             //you can use tools like Netron, 
             // which is installed by Visual Studio AI Tools
 
@@ -43,139 +42,46 @@ public struct TinyYoloModelSettings
             public const string ModelOutput = "grid";
         }
 
-        public void Score()
-        {
-            var model = LoadModel(modelLocation);
-
-            PredictDataUsingModel(imagesFolder, model);
-        }
-
-        private PredictionEngine<ImageNetData, ImageNetPrediction> LoadModel(string modelLocation)
+        private ITransformer LoadModel(string modelLocation)
         {
             Console.WriteLine("Read model");
             Console.WriteLine($"Model location: {modelLocation}");
             Console.WriteLine($"Default parameters: image size=({ImageNetSettings.imageWidth},{ImageNetSettings.imageHeight})");
 
-            var data = CreateEmptyDataView();
+            // Create IDataView from empty list to obtain input data schema
+            var data = mlContext.Data.LoadFromEnumerable(new List<ImageNetData>());
 
+            // Define scoring pipeline
             var pipeline = mlContext.Transforms.LoadImages(outputColumnName: "image", imageFolder: "", inputColumnName: nameof(ImageNetData.ImagePath))
                             .Append(mlContext.Transforms.ResizeImages(outputColumnName: "image", imageWidth: ImageNetSettings.imageWidth, imageHeight: ImageNetSettings.imageHeight, inputColumnName: "image"))
                             .Append(mlContext.Transforms.ExtractPixels(outputColumnName: "image"))
                             .Append(mlContext.Transforms.ApplyOnnxModel(modelFile: modelLocation, outputColumnNames: new[] { TinyYoloModelSettings.ModelOutput }, inputColumnNames: new[] { TinyYoloModelSettings.ModelInput }));
 
+            // Fit scoring pipeline
             var model = pipeline.Fit(data);
 
-            var predictionEngine = mlContext.Model.CreatePredictionEngine<ImageNetData, ImageNetPrediction>(model);
-
-            return predictionEngine;
+            return model;
         }
 
-        protected void PredictDataUsingModel(string imagesFolder, PredictionEngine<ImageNetData, ImageNetPrediction> model)
+        private IEnumerable<float[]> PredictDataUsingModel(IDataView testData, ITransformer model)
         {
             Console.WriteLine($"Images location: {imagesFolder}");
             Console.WriteLine("");
             Console.WriteLine("=====Identify the objects in the images=====");
             Console.WriteLine("");
 
-            var testData = GetImagesData(imagesFolder);
-
-            foreach (var sample in testData)
-            {
-                var probs = model.Predict(sample).PredictedLabels;
-                _boundingBoxes = _parser.ParseOutputs(probs);
-                var filteredBoxes = _parser.FilterBoundingBoxes(_boundingBoxes, 5, .5F);
+            IDataView scoredData = model.Transform(testData);
 
+            IEnumerable<float[]> probabilities = scoredData.GetColumn<float[]>(TinyYoloModelSettings.ModelOutput);
 
-                var outputDirectory = Path.Combine(Directory.GetParent(sample.ImagePath).FullName, "output");
-                var filename = new FileInfo(sample.ImagePath).Name;
-
-                DrawBoundingBox(imagesFolder, outputDirectory, filename, filteredBoxes);
-
-                Console.WriteLine(".....The objects in the image {0} are detected as below....", sample.Label);
-                foreach (var box in filteredBoxes)
-                {
-                    Console.WriteLine(box.Label + " and its Confidence score: " + box.Confidence);
-                }
-                Console.WriteLine("");
-            }
+            return probabilities;
         }
 
-        private static IEnumerable<ImageNetData> GetImagesData(string folder)
-        {
-            List<ImageNetData> imagesList = new List<ImageNetData>();
-            string[] filePaths = Directory.GetFiles(folder).Where(filePath => Path.GetExtension(filePath) != ".md").ToArray();
-            foreach (var filePath in filePaths)
-            {
-                ImageNetData imagedata = new ImageNetData { ImagePath = filePath, Label = Path.GetFileName(filePath) };
-                imagesList.Add(imagedata);
-            }
-            return imagesList;
-        }
-
-        private IDataView CreateEmptyDataView()
+        public IEnumerable<float[]> Score(IDataView data)
         {
-            //Create empty DataView. We just need the schema to call fit()
-            List<ImageNetData> list = new List<ImageNetData>();
-            IEnumerable<ImageNetData> enumerableData = list;
-            var dv = mlContext.Data.LoadFromEnumerable(enumerableData);
-            return dv;
-        }
+            var model = LoadModel(modelLocation);
 
-        public void DrawBoundingBox(string inputImageLocation, string outputImageLocation, string imageName, IList<YoloBoundingBox> filteredBoundingBoxes)
-        {
-            Image image = Image.FromFile(Path.Combine(inputImageLocation, imageName));
-
-            var originalImageHeight = image.Height;
-            var originalImageWidth = image.Width;
-
-            foreach (var box in filteredBoundingBoxes)
-            {
-                // Get Bounding Box Dimensions
-                var x = (uint)Math.Max(box.Dimensions.X, 0);
-                var y = (uint)Math.Max(box.Dimensions.Y, 0);
-                var width = (uint)Math.Min(originalImageWidth - x, box.Dimensions.Width);
-                var height = (uint)Math.Min(originalImageHeight - y, box.Dimensions.Height);
-
-                // Resize To Image
-                x = (uint)originalImageWidth * x / 416;
-                y = (uint)originalImageHeight * y / 416;
-                width = (uint)originalImageWidth * width / 416;
-                height = (uint)originalImageHeight * height / 416;
-
-                // Bounding Box Text
-                string text = $"{box.Label} ({(box.Confidence * 100).ToString("0")}%)";
-
-                using (Graphics thumbnailGraphic = Graphics.FromImage(image))
-                {
-                    thumbnailGraphic.CompositingQuality = CompositingQuality.HighQuality;
-                    thumbnailGraphic.SmoothingMode = SmoothingMode.HighQuality;
-                    thumbnailGraphic.InterpolationMode = InterpolationMode.HighQualityBicubic;
-
-                    // Define Text Options
-                    Font drawFont = new Font("Arial", 12, FontStyle.Bold);
-                    SizeF size = thumbnailGraphic.MeasureString(text, drawFont);
-                    SolidBrush fontBrush = new SolidBrush(Color.Black);
-                    Point atPoint = new Point((int)x, (int)y - (int)size.Height - 1);
-
-                    // Define BoundingBox options
-                    Pen pen = new Pen(box.BoxColor, 3.2f);
-                    SolidBrush colorBrush = new SolidBrush(box.BoxColor);
-
-                    // Draw text on image 
-                    thumbnailGraphic.FillRectangle(colorBrush, (int)x, (int)(y - size.Height - 1), (int)size.Width, (int)size.Height);
-                    thumbnailGraphic.DrawString(text, drawFont, fontBrush, atPoint);
-
-                    // Draw bounding box on image
-                    thumbnailGraphic.DrawRectangle(pen, x, y, width, height);
-                }
-            }
-
-            if (!Directory.Exists(outputImageLocation))
-            {
-                Directory.CreateDirectory(outputImageLocation);
-            }
-
-            image.Save(Path.Combine(outputImageLocation, imageName));
+            return PredictDataUsingModel(data, model);
         }
     }
 }

samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs

@@ -1,5 +1,12 @@
 using System;
 using System.IO;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Drawing.Drawing2D;
+using System.Linq;
+using Microsoft.ML;
+using ObjectDetection.YoloParser;
+using ObjectDetection.DataStructures;
 
 namespace ObjectDetection
 {
@@ -11,11 +18,41 @@ public static void Main()
             string assetsPath = GetAbsolutePath(assetsRelativePath);
             var modelFilePath = Path.Combine(assetsPath, "Model", "TinyYolo2_model.onnx");
             var imagesFolder = Path.Combine(assetsPath, "images");
+            var outputFolder = Path.Combine(assetsPath, "images", "output");
+
+            // Initialize MLContext
+            MLContext mlContext = new MLContext();
 
             try
             {
-                var modelScorer = new OnnxModelScorer(imagesFolder, modelFilePath);
-                modelScorer.Score();
+                // Load Data
+                IEnumerable<ImageNetData> images = ImageNetData.ReadFromFile(imagesFolder);
+                IDataView imageDataView = mlContext.Data.LoadFromEnumerable(images);
+
+                // Create instance of model scorer
+                var modelScorer = new OnnxModelScorer(imagesFolder, modelFilePath, mlContext);
+
+                // Use model to score data
+                IEnumerable<float[]> probabilities = modelScorer.Score(imageDataView);
+
+                // Post-process model output
+                YoloWinMlParser parser = new YoloWinMlParser();
+
+                var boundingBoxes =
+                    probabilities
+                    .Select(probability => parser.ParseOutputs(probability))
+                    .Select(boxes => parser.FilterBoundingBoxes(boxes, 5, .5F));
+
+                // Draw bounding boxes for detected objects in each of the images
+                for (var i = 0; i < images.Count(); i++)
+                {
+                    string imageFileName = images.ElementAt(i).Label;
+                    IList<YoloBoundingBox> detectedObjects = boundingBoxes.ElementAt(i);
+
+                    DrawBoundingBox(imagesFolder, outputFolder, imageFileName, detectedObjects);
+
+                    LogDetectedObjects(imageFileName, detectedObjects);
+                }
             }
             catch (Exception ex)
             {
@@ -35,6 +72,75 @@ public static string GetAbsolutePath(string relativePath)
 
             return fullPath;
         }
+
+        private static void DrawBoundingBox(string inputImageLocation, string outputImageLocation, string imageName, IList<YoloBoundingBox> filteredBoundingBoxes)
+        {
+            Image image = Image.FromFile(Path.Combine(inputImageLocation, imageName));
+
+            var originalImageHeight = image.Height;
+            var originalImageWidth = image.Width;
+
+            foreach (var box in filteredBoundingBoxes)
+            {
+                // Get Bounding Box Dimensions
+                var x = (uint)Math.Max(box.Dimensions.X, 0);
+                var y = (uint)Math.Max(box.Dimensions.Y, 0);
+                var width = (uint)Math.Min(originalImageWidth - x, box.Dimensions.Width);
+                var height = (uint)Math.Min(originalImageHeight - y, box.Dimensions.Height);
+
+                // Resize To Image
+                x = (uint)originalImageWidth * x / OnnxModelScorer.ImageNetSettings.imageWidth;
+                y = (uint)originalImageHeight * y / OnnxModelScorer.ImageNetSettings.imageHeight;
+                width = (uint)originalImageWidth * width / OnnxModelScorer.ImageNetSettings.imageWidth;
+                height = (uint)originalImageHeight * height / OnnxModelScorer.ImageNetSettings.imageHeight;
+
+                // Bounding Box Text
+                string text = $"{box.Label} ({(box.Confidence * 100).ToString("0")}%)";
+
+                using (Graphics thumbnailGraphic = Graphics.FromImage(image))
+                {
+                    thumbnailGraphic.CompositingQuality = CompositingQuality.HighQuality;
+                    thumbnailGraphic.SmoothingMode = SmoothingMode.HighQuality;
+                    thumbnailGraphic.InterpolationMode = InterpolationMode.HighQualityBicubic;
+
+                    // Define Text Options
+                    Font drawFont = new Font("Arial", 12, FontStyle.Bold);
+                    SizeF size = thumbnailGraphic.MeasureString(text, drawFont);
+                    SolidBrush fontBrush = new SolidBrush(Color.Black);
+                    Point atPoint = new Point((int)x, (int)y - (int)size.Height - 1);
+
+                    // Define BoundingBox options
+                    Pen pen = new Pen(box.BoxColor, 3.2f);
+                    SolidBrush colorBrush = new SolidBrush(box.BoxColor);
+
+                    // Draw text on image 
+                    thumbnailGraphic.FillRectangle(colorBrush, (int)x, (int)(y - size.Height - 1), (int)size.Width, (int)size.Height);
+                    thumbnailGraphic.DrawString(text, drawFont, fontBrush, atPoint);
+
+                    // Draw bounding box on image
+                    thumbnailGraphic.DrawRectangle(pen, x, y, width, height);
+                }
+            }
+
+            if (!Directory.Exists(outputImageLocation))
+            {
+                Directory.CreateDirectory(outputImageLocation);
+            }
+
+            image.Save(Path.Combine(outputImageLocation, imageName));
+        }
+
+        private static void LogDetectedObjects(string imageName, IList<YoloBoundingBox> boundingBoxes)
+        {
+            Console.WriteLine($".....The objects in the image {imageName} are detected as below....");
+
+            foreach (var box in boundingBoxes)
+            {
+                Console.WriteLine($"{box.Label} and its Confidence score: {box.Confidence}");
+            }
+
+            Console.WriteLine("");
+        }
     }
 }
 

samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloBoundingBox.cs

@@ -1,9 +1,10 @@
-using ObjectDetection.YoloParser;
-using System.Drawing;
+using System.Drawing;
 
-namespace ObjectDetection
+namespace ObjectDetection.YoloParser
 {
-    class YoloBoundingBox
+    public class BoundingBoxDimensions : DimensionsBase { }
+
+    public class YoloBoundingBox
     {
         public BoundingBoxDimensions Dimensions { get; set; }
 
@@ -18,6 +19,5 @@ public RectangleF Rect
 
         public Color BoxColor { get; set; }
     }
-
-    class BoundingBoxDimensions : DimensionsBase { }
+
 }