From 9d6cf64768d18ca1574fbe6905e239e739cd4633 Mon Sep 17 00:00:00 2001 From: Luis Quintanilla Date: Fri, 21 Jun 2019 16:47:54 -0700 Subject: [PATCH 1/8] Initial commit --- .../ObjectDetectionConsoleApp/OnnxModelScorer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs index 28ca0e159..8cea73c2c 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs @@ -121,7 +121,7 @@ private IDataView CreateEmptyDataView() return dv; } - public void DrawBoundingBox(string inputImageLocation, string outputImageLocation, string imageName, IList filteredBoundingBoxes) + private void DrawBoundingBox(string inputImageLocation, string outputImageLocation, string imageName, IList filteredBoundingBoxes) { Image image = Image.FromFile(Path.Combine(inputImageLocation, imageName)); From dd2d728af069733ed4590bee39af75c8c2e4fba2 Mon Sep 17 00:00:00 2001 From: Luis Quintanilla Date: Wed, 26 Jun 2019 01:16:16 -0400 Subject: [PATCH 2/8] Refactored for flow --- .../DataStructures/ImageNetData.cs | 19 ++- .../DataStructures/ImageNetPrediction.cs | 4 +- .../OnnxModelScorer.cs | 149 +++--------------- .../ObjectDetectionConsoleApp/Program.cs | 110 ++++++++++++- .../YoloParser/YoloBoundingBox.cs | 12 +- ...YoloWinMlParser.cs => YoloOutputParser.cs} | 18 +-- 6 files changed, 162 insertions(+), 150 deletions(-) rename samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/{YoloWinMlParser.cs => YoloOutputParser.cs} (93%) diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetData.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetData.cs index 62c316f7d..c26ac6464 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetData.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetData.cs @@ -1,6 +1,9 @@ -using Microsoft.ML.Data; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.ML.Data; -namespace ObjectDetection +namespace ObjectDetection.DataStructures { public class ImageNetData { @@ -9,11 +12,13 @@ public class ImageNetData [LoadColumn(1)] public string Label; - } - public class ImageNetDataProbability : ImageNetData - { - public string PredictedLabel; - public float Probability { get; set; } + public static IEnumerable LoadImageNetData(string imageFolder) + { + return Directory + .GetFiles(imageFolder) + .Where(filePath => Path.GetExtension(filePath) != ".md") + .Select(filePath => new ImageNetData { ImagePath = filePath, Label = Path.GetFileName(filePath) }); + } } } \ No newline at end of file diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetPrediction.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetPrediction.cs index 2bfc27a8f..374ecb7f8 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetPrediction.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetPrediction.cs @@ -1,10 +1,10 @@ using Microsoft.ML.Data; -namespace ObjectDetection +namespace ObjectDetection.DataStructures { public class ImageNetPrediction { - [ColumnName(OnnxModelScorer.TinyYoloModelSettings.ModelOutput)] + [ColumnName("grid")] public float[] PredictedLabels; } } diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs index 8cea73c2c..6e5800e7e 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs @@ -1,27 +1,26 @@ using System; using System.Collections.Generic; -using System.Drawing; -using System.Drawing.Drawing2D; -using System.IO; using System.Linq; using Microsoft.ML; +using Microsoft.ML.Data; +using ObjectDetection.DataStructures; +using ObjectDetection.YoloParser; namespace ObjectDetection { class OnnxModelScorer { - private readonly string imagesFolder; - private readonly string modelLocation; - private readonly MLContext mlContext; + private readonly string _imagesFolder; + private readonly string _modelLocation; + private readonly MLContext _mlContext; private IList _boundingBoxes = new List(); - private readonly YoloWinMlParser _parser = new YoloWinMlParser(); - public OnnxModelScorer(string imagesFolder, string modelLocation) + public OnnxModelScorer(string imagesFolder, string modelLocation, MLContext mlContext) { - this.imagesFolder = imagesFolder; - this.modelLocation = modelLocation; - mlContext = new MLContext(); + _imagesFolder = imagesFolder; + _modelLocation = modelLocation; + _mlContext = mlContext; } public struct ImageNetSettings @@ -32,7 +31,7 @@ public struct ImageNetSettings public struct TinyYoloModelSettings { - // for checking TIny yolo2 Model input and output parameter names, + // for checking Tiny yolo2 Model input and output parameter names, //you can use tools like Netron, // which is installed by Visual Studio AI Tools @@ -43,139 +42,43 @@ public struct TinyYoloModelSettings public const string ModelOutput = "grid"; } - public void Score() - { - var model = LoadModel(modelLocation); - - PredictDataUsingModel(imagesFolder, model); - } - - private PredictionEngine LoadModel(string modelLocation) + private ITransformer LoadModel(string modelLocation) { Console.WriteLine("Read model"); Console.WriteLine($"Model location: {modelLocation}"); Console.WriteLine($"Default parameters: image size=({ImageNetSettings.imageWidth},{ImageNetSettings.imageHeight})"); - var data = CreateEmptyDataView(); + var data = _mlContext.Data.LoadFromEnumerable(new List()); - var pipeline = mlContext.Transforms.LoadImages(outputColumnName: "image", imageFolder: "", inputColumnName: nameof(ImageNetData.ImagePath)) - .Append(mlContext.Transforms.ResizeImages(outputColumnName: "image", imageWidth: ImageNetSettings.imageWidth, imageHeight: ImageNetSettings.imageHeight, inputColumnName: "image")) - .Append(mlContext.Transforms.ExtractPixels(outputColumnName: "image")) - .Append(mlContext.Transforms.ApplyOnnxModel(modelFile: modelLocation, outputColumnNames: new[] { TinyYoloModelSettings.ModelOutput }, inputColumnNames: new[] { TinyYoloModelSettings.ModelInput })); + var pipeline = _mlContext.Transforms.LoadImages(outputColumnName: "image", imageFolder: "", inputColumnName: nameof(ImageNetData.ImagePath)) + .Append(_mlContext.Transforms.ResizeImages(outputColumnName: "image", imageWidth: ImageNetSettings.imageWidth, imageHeight: ImageNetSettings.imageHeight, inputColumnName: "image")) + .Append(_mlContext.Transforms.ExtractPixels(outputColumnName: "image")) + .Append(_mlContext.Transforms.ApplyOnnxModel(modelFile: modelLocation, outputColumnNames: new[] { TinyYoloModelSettings.ModelOutput }, inputColumnNames: new[] { TinyYoloModelSettings.ModelInput })); var model = pipeline.Fit(data); - var predictionEngine = mlContext.Model.CreatePredictionEngine(model); - - return predictionEngine; + return model; } - protected void PredictDataUsingModel(string imagesFolder, PredictionEngine model) + private IEnumerable PredictDataUsingModel(IDataView testData, ITransformer model) { - Console.WriteLine($"Images location: {imagesFolder}"); + Console.WriteLine($"Images location: {_imagesFolder}"); Console.WriteLine(""); Console.WriteLine("=====Identify the objects in the images====="); Console.WriteLine(""); - var testData = GetImagesData(imagesFolder); - - foreach (var sample in testData) - { - var probs = model.Predict(sample).PredictedLabels; - _boundingBoxes = _parser.ParseOutputs(probs); - var filteredBoxes = _parser.FilterBoundingBoxes(_boundingBoxes, 5, .5F); + IDataView scoredData = model.Transform(testData); + IEnumerable probabilities = scoredData.GetColumn(TinyYoloModelSettings.ModelOutput); - var outputDirectory = Path.Combine(Directory.GetParent(sample.ImagePath).FullName, "output"); - var filename = new FileInfo(sample.ImagePath).Name; - - DrawBoundingBox(imagesFolder, outputDirectory, filename, filteredBoxes); - - Console.WriteLine(".....The objects in the image {0} are detected as below....", sample.Label); - foreach (var box in filteredBoxes) - { - Console.WriteLine(box.Label + " and its Confidence score: " + box.Confidence); - } - Console.WriteLine(""); - } + return probabilities; } - private static IEnumerable GetImagesData(string folder) - { - List imagesList = new List(); - string[] filePaths = Directory.GetFiles(folder).Where(filePath => Path.GetExtension(filePath) != ".md").ToArray(); - foreach (var filePath in filePaths) - { - ImageNetData imagedata = new ImageNetData { ImagePath = filePath, Label = Path.GetFileName(filePath) }; - imagesList.Add(imagedata); - } - return imagesList; - } - - private IDataView CreateEmptyDataView() + public IEnumerable Score(IDataView data) { - //Create empty DataView. We just need the schema to call fit() - List list = new List(); - IEnumerable enumerableData = list; - var dv = mlContext.Data.LoadFromEnumerable(enumerableData); - return dv; - } + var model = LoadModel(_modelLocation); - private void DrawBoundingBox(string inputImageLocation, string outputImageLocation, string imageName, IList filteredBoundingBoxes) - { - Image image = Image.FromFile(Path.Combine(inputImageLocation, imageName)); - - var originalImageHeight = image.Height; - var originalImageWidth = image.Width; - - foreach (var box in filteredBoundingBoxes) - { - // Get Bounding Box Dimensions - var x = (uint)Math.Max(box.Dimensions.X, 0); - var y = (uint)Math.Max(box.Dimensions.Y, 0); - var width = (uint)Math.Min(originalImageWidth - x, box.Dimensions.Width); - var height = (uint)Math.Min(originalImageHeight - y, box.Dimensions.Height); - - // Resize To Image - x = (uint)originalImageWidth * x / 416; - y = (uint)originalImageHeight * y / 416; - width = (uint)originalImageWidth * width / 416; - height = (uint)originalImageHeight * height / 416; - - // Bounding Box Text - string text = $"{box.Label} ({(box.Confidence * 100).ToString("0")}%)"; - - using (Graphics thumbnailGraphic = Graphics.FromImage(image)) - { - thumbnailGraphic.CompositingQuality = CompositingQuality.HighQuality; - thumbnailGraphic.SmoothingMode = SmoothingMode.HighQuality; - thumbnailGraphic.InterpolationMode = InterpolationMode.HighQualityBicubic; - - // Define Text Options - Font drawFont = new Font("Arial", 12, FontStyle.Bold); - SizeF size = thumbnailGraphic.MeasureString(text, drawFont); - SolidBrush fontBrush = new SolidBrush(Color.Black); - Point atPoint = new Point((int)x, (int)y - (int)size.Height - 1); - - // Define BoundingBox options - Pen pen = new Pen(box.BoxColor, 3.2f); - SolidBrush colorBrush = new SolidBrush(box.BoxColor); - - // Draw text on image - thumbnailGraphic.FillRectangle(colorBrush, (int)x, (int)(y - size.Height - 1), (int)size.Width, (int)size.Height); - thumbnailGraphic.DrawString(text, drawFont, fontBrush, atPoint); - - // Draw bounding box on image - thumbnailGraphic.DrawRectangle(pen, x, y, width, height); - } - } - - if (!Directory.Exists(outputImageLocation)) - { - Directory.CreateDirectory(outputImageLocation); - } - - image.Save(Path.Combine(outputImageLocation, imageName)); + return PredictDataUsingModel(data, model); } } } diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs index 93745428f..dc110b1cb 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs @@ -1,5 +1,12 @@ using System; using System.IO; +using System.Collections.Generic; +using System.Drawing; +using System.Drawing.Drawing2D; +using System.Linq; +using Microsoft.ML; +using ObjectDetection.YoloParser; +using ObjectDetection.DataStructures; namespace ObjectDetection { @@ -11,11 +18,41 @@ public static void Main() string assetsPath = GetAbsolutePath(assetsRelativePath); var modelFilePath = Path.Combine(assetsPath, "Model", "TinyYolo2_model.onnx"); var imagesFolder = Path.Combine(assetsPath, "images"); + var outputFolder = Path.Combine(assetsPath, "images", "output"); + + // Initialize MLContext + MLContext mlContext = new MLContext(); try { - var modelScorer = new OnnxModelScorer(imagesFolder, modelFilePath); - modelScorer.Score(); + // Load Data + IEnumerable images = ImageNetData.LoadImageNetData(imagesFolder); + IDataView imageDataView = mlContext.Data.LoadFromEnumerable(images); + + // Create instance of model scorer + var modelScorer = new OnnxModelScorer(imagesFolder, modelFilePath, mlContext); + + // Use model to score data + IEnumerable probabilities = modelScorer.Score(imageDataView); + + // Post-process model output + YoloOutputParser parser = new YoloOutputParser(); + + var boundingBoxes = + probabilities + .Select(probability => parser.ParseOutputs(probability)) + .Select(boxes => parser.FilterBoundingBoxes(boxes, 5, .5F)); + + // Draw bounding boxes for detected objects in each of the images + for (var i = 0; i < images.Count(); i++) + { + string imageFileName = images.ElementAt(i).Label; + IList detectedObjects = boundingBoxes.ElementAt(i); + + DrawBoundingBox(imagesFolder, outputFolder, imageFileName, detectedObjects); + + LogDetectedObjects(imageFileName, detectedObjects); + } } catch (Exception ex) { @@ -35,6 +72,75 @@ public static string GetAbsolutePath(string relativePath) return fullPath; } + + private static void DrawBoundingBox(string inputImageLocation, string outputImageLocation, string imageName, IList filteredBoundingBoxes) + { + Image image = Image.FromFile(Path.Combine(inputImageLocation, imageName)); + + var originalImageHeight = image.Height; + var originalImageWidth = image.Width; + + foreach (var box in filteredBoundingBoxes) + { + // Get Bounding Box Dimensions + var x = (uint)Math.Max(box.Dimensions.X, 0); + var y = (uint)Math.Max(box.Dimensions.Y, 0); + var width = (uint)Math.Min(originalImageWidth - x, box.Dimensions.Width); + var height = (uint)Math.Min(originalImageHeight - y, box.Dimensions.Height); + + // Resize To Image + x = (uint)originalImageWidth * x / 416; + y = (uint)originalImageHeight * y / 416; + width = (uint)originalImageWidth * width / 416; + height = (uint)originalImageHeight * height / 416; + + // Bounding Box Text + string text = $"{box.Label} ({(box.Confidence * 100).ToString("0")}%)"; + + using (Graphics thumbnailGraphic = Graphics.FromImage(image)) + { + thumbnailGraphic.CompositingQuality = CompositingQuality.HighQuality; + thumbnailGraphic.SmoothingMode = SmoothingMode.HighQuality; + thumbnailGraphic.InterpolationMode = InterpolationMode.HighQualityBicubic; + + // Define Text Options + Font drawFont = new Font("Arial", 12, FontStyle.Bold); + SizeF size = thumbnailGraphic.MeasureString(text, drawFont); + SolidBrush fontBrush = new SolidBrush(Color.Black); + Point atPoint = new Point((int)x, (int)y - (int)size.Height - 1); + + // Define BoundingBox options + Pen pen = new Pen(box.BoxColor, 3.2f); + SolidBrush colorBrush = new SolidBrush(box.BoxColor); + + // Draw text on image + thumbnailGraphic.FillRectangle(colorBrush, (int)x, (int)(y - size.Height - 1), (int)size.Width, (int)size.Height); + thumbnailGraphic.DrawString(text, drawFont, fontBrush, atPoint); + + // Draw bounding box on image + thumbnailGraphic.DrawRectangle(pen, x, y, width, height); + } + } + + if (!Directory.Exists(outputImageLocation)) + { + Directory.CreateDirectory(outputImageLocation); + } + + image.Save(Path.Combine(outputImageLocation, imageName)); + } + + private static void LogDetectedObjects(string imageName, IList boundingBoxes) + { + Console.WriteLine($".....The objects in the image {imageName} are detected as below...."); + + foreach (var box in boundingBoxes) + { + Console.WriteLine($"{box.Label} and its Confidence score: {box.Confidence}"); + } + + Console.WriteLine(""); + } } } diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloBoundingBox.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloBoundingBox.cs index 28b96f5af..31397dbb1 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloBoundingBox.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloBoundingBox.cs @@ -1,9 +1,10 @@ -using ObjectDetection.YoloParser; -using System.Drawing; +using System.Drawing; -namespace ObjectDetection +namespace ObjectDetection.YoloParser { - class YoloBoundingBox + public class BoundingBoxDimensions : DimensionsBase { } + + public class YoloBoundingBox { public BoundingBoxDimensions Dimensions { get; set; } @@ -18,6 +19,5 @@ public RectangleF Rect public Color BoxColor { get; set; } } - - class BoundingBoxDimensions : DimensionsBase { } + } \ No newline at end of file diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloWinMlParser.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs similarity index 93% rename from samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloWinMlParser.cs rename to samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs index 2bad90a99..5a59404b6 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloWinMlParser.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs @@ -2,13 +2,11 @@ using System.Collections.Generic; using System.Drawing; using System.Linq; -using ObjectDetection.YoloParser; -namespace ObjectDetection +namespace ObjectDetection.YoloParser { - class YoloWinMlParser + class YoloOutputParser { - class CellDimensions : DimensionsBase { } public const int ROW_COUNT = 13; @@ -79,7 +77,7 @@ public IList ParseOutputs(float[] yoloModelOutputs, float thres float confidence = GetConfidence(yoloModelOutputs, row, column, channel); - CellDimensions mappedBoundingBoxes = MapBoundingBoxToCell(row, column, box, boundingBoxDimensions); + CellDimensions mappedBoundingBox = MapBoundingBoxToCell(row, column, box, boundingBoxDimensions); if (confidence < threshold) continue; @@ -96,10 +94,10 @@ public IList ParseOutputs(float[] yoloModelOutputs, float thres { Dimensions = new BoundingBoxDimensions { - X = (mappedBoundingBoxes.X - mappedBoundingBoxes.Width / 2), - Y = (mappedBoundingBoxes.Y - mappedBoundingBoxes.Height / 2), - Width = mappedBoundingBoxes.Width, - Height = mappedBoundingBoxes.Height, + X = (mappedBoundingBox.X - mappedBoundingBox.Width / 2), + Y = (mappedBoundingBox.Y - mappedBoundingBox.Height / 2), + Width = mappedBoundingBox.Width, + Height = mappedBoundingBox.Height, }, Confidence = topScore, Label = labels[topResultIndex], @@ -221,7 +219,7 @@ public float[] ExtractClasses(float[] modelOutput, int x, int y, int channel) return Softmax(predictedClasses); } - public ValueTuple GetTopResult(float[] predictedClasses) + private ValueTuple GetTopResult(float[] predictedClasses) { return predictedClasses .Select((predictedClass, index) => (Index: index, Value: predictedClass)) From 97f3a33dc5bb15ebb7c25a4f338731e225670bd7 Mon Sep 17 00:00:00 2001 From: Luis Quintanilla Date: Wed, 26 Jun 2019 01:49:17 -0400 Subject: [PATCH 3/8] Renamed image read method for clarity --- .../ObjectDetectionConsoleApp/DataStructures/ImageNetData.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetData.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetData.cs index c26ac6464..c028e0924 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetData.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/DataStructures/ImageNetData.cs @@ -13,7 +13,7 @@ public class ImageNetData [LoadColumn(1)] public string Label; - public static IEnumerable LoadImageNetData(string imageFolder) + public static IEnumerable ReadFromFile(string imageFolder) { return Directory .GetFiles(imageFolder) From d31179a9137d031c7e5282bbf2ff26fd0ce5d018 Mon Sep 17 00:00:00 2001 From: Luis Quintanilla Date: Wed, 26 Jun 2019 03:53:37 -0400 Subject: [PATCH 4/8] Rearranged methods based on the order used --- .../ObjectDetectionConsoleApp/Program.cs | 2 +- .../YoloParser/YoloOutputParser.cs | 183 +++++++++--------- 2 files changed, 93 insertions(+), 92 deletions(-) diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs index dc110b1cb..d9f745a8f 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs @@ -26,7 +26,7 @@ public static void Main() try { // Load Data - IEnumerable images = ImageNetData.LoadImageNetData(imagesFolder); + IEnumerable images = ImageNetData.ReadFromFile(imagesFolder); IDataView imageDataView = mlContext.Data.LoadFromEnumerable(images); // Create instance of model scorer diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs index 5a59404b6..5cd063d9b 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs @@ -58,6 +58,98 @@ class CellDimensions : DimensionsBase { } Color.DarkTurquoise }; + private float Sigmoid(float value) + { + var k = (float)Math.Exp(value); + return k / (1.0f + k); + } + + private float[] Softmax(float[] values) + { + var maxVal = values.Max(); + var exp = values.Select(v => Math.Exp(v - maxVal)); + var sumExp = exp.Sum(); + + return exp.Select(v => (float)(v / sumExp)).ToArray(); + } + + private int GetOffset(int x, int y, int channel) + { + // YOLO outputs a tensor that has a shape of 125x13x13, which + // WinML flattens into a 1D array. To access a specific channel + // for a given (x,y) cell position, we need to calculate an offset + // into the array + return (channel * this.channelStride) + (y * COL_COUNT) + x; + } + + private BoundingBoxDimensions ExtractBoundingBoxDimensions(float[] modelOutput, int x, int y, int channel) + { + return new BoundingBoxDimensions + { + X = modelOutput[GetOffset(x, y, channel)], + Y = modelOutput[GetOffset(x, y, channel + 1)], + Width = modelOutput[GetOffset(x, y, channel + 2)], + Height = modelOutput[GetOffset(x, y, channel + 3)] + }; + } + + private float GetConfidence(float[] modelOutput, int x, int y, int channel) + { + return Sigmoid(modelOutput[GetOffset(x, y, channel + 4)]); + } + + private CellDimensions MapBoundingBoxToCell(int x, int y, int box, BoundingBoxDimensions boxDimensions) + { + return new CellDimensions + { + X = ((float)x + Sigmoid(boxDimensions.X)) * CELL_WIDTH, + Y = ((float)y + Sigmoid(boxDimensions.Y)) * CELL_HEIGHT, + Width = (float)Math.Exp(boxDimensions.Width) * CELL_WIDTH * anchors[box * 2], + Height = (float)Math.Exp(boxDimensions.Height) * CELL_HEIGHT * anchors[box * 2 + 1], + }; + } + + public float[] ExtractClasses(float[] modelOutput, int x, int y, int channel) + { + float[] predictedClasses = new float[CLASS_COUNT]; + int predictedClassOffset = channel + BOX_INFO_FEATURE_COUNT; + for (int predictedClass = 0; predictedClass < CLASS_COUNT; predictedClass++) + { + predictedClasses[predictedClass] = modelOutput[GetOffset(x, y, predictedClass + predictedClassOffset)]; + } + return Softmax(predictedClasses); + } + + private ValueTuple GetTopResult(float[] predictedClasses) + { + return predictedClasses + .Select((predictedClass, index) => (Index: index, Value: predictedClass)) + .OrderByDescending(result => result.Value) + .First(); + } + + private float IntersectionOverUnion(RectangleF boundingBoxA, RectangleF boundingBoxB) + { + var areaA = boundingBoxA.Width * boundingBoxA.Height; + + if (areaA <= 0) + return 0; + + var areaB = boundingBoxB.Width * boundingBoxB.Height; + + if (areaB <= 0) + return 0; + + var minX = Math.Max(boundingBoxA.Left, boundingBoxB.Left); + var minY = Math.Max(boundingBoxA.Top, boundingBoxB.Top); + var maxX = Math.Min(boundingBoxA.Right, boundingBoxB.Right); + var maxY = Math.Min(boundingBoxA.Bottom, boundingBoxB.Bottom); + + var intersectionArea = Math.Max(maxY - minY, 0) * Math.Max(maxX - minX, 0); + + return intersectionArea / (areaA + areaB - intersectionArea); + } + public IList ParseOutputs(float[] yoloModelOutputs, float threshold = .3F) { var boxes = new List(); @@ -157,96 +249,5 @@ public IList FilterBoundingBoxes(IList boxes, return results; } - private float Sigmoid(float value) - { - var k = (float)Math.Exp(value); - return k / (1.0f + k); - } - - private float[] Softmax(float[] values) - { - var maxVal = values.Max(); - var exp = values.Select(v => Math.Exp(v - maxVal)); - var sumExp = exp.Sum(); - - return exp.Select(v => (float)(v / sumExp)).ToArray(); - } - - private int GetOffset(int x, int y, int channel) - { - // YOLO outputs a tensor that has a shape of 125x13x13, which - // WinML flattens into a 1D array. To access a specific channel - // for a given (x,y) cell position, we need to calculate an offset - // into the array - return (channel * this.channelStride) + (y * COL_COUNT) + x; - } - - private BoundingBoxDimensions ExtractBoundingBoxDimensions(float[] modelOutput, int x, int y, int channel) - { - return new BoundingBoxDimensions - { - X = modelOutput[GetOffset(x, y, channel)], - Y = modelOutput[GetOffset(x, y, channel + 1)], - Width = modelOutput[GetOffset(x, y, channel + 2)], - Height = modelOutput[GetOffset(x, y, channel + 3)] - }; - } - - private float GetConfidence(float[] modelOutput, int x, int y, int channel) - { - return Sigmoid(modelOutput[GetOffset(x, y, channel + 4)]); - } - - private CellDimensions MapBoundingBoxToCell(int x, int y, int box, BoundingBoxDimensions boxDimensions) - { - return new CellDimensions - { - X = ((float)x + Sigmoid(boxDimensions.X)) * CELL_WIDTH, - Y = ((float)y + Sigmoid(boxDimensions.Y)) * CELL_HEIGHT, - Width = (float)Math.Exp(boxDimensions.Width) * CELL_WIDTH * anchors[box * 2], - Height = (float)Math.Exp(boxDimensions.Height) * CELL_HEIGHT * anchors[box * 2 + 1], - }; - } - - public float[] ExtractClasses(float[] modelOutput, int x, int y, int channel) - { - float[] predictedClasses = new float[CLASS_COUNT]; - int predictedClassOffset = channel + BOX_INFO_FEATURE_COUNT; - for (int predictedClass = 0; predictedClass < CLASS_COUNT; predictedClass++) - { - predictedClasses[predictedClass] = modelOutput[GetOffset(x, y, predictedClass + predictedClassOffset)]; - } - return Softmax(predictedClasses); - } - - private ValueTuple GetTopResult(float[] predictedClasses) - { - return predictedClasses - .Select((predictedClass, index) => (Index: index, Value: predictedClass)) - .OrderByDescending(result => result.Value) - .First(); - } - - private float IntersectionOverUnion(RectangleF boundingBoxA, RectangleF boundingBoxB) - { - var areaA = boundingBoxA.Width * boundingBoxA.Height; - - if (areaA <= 0) - return 0; - - var areaB = boundingBoxB.Width * boundingBoxB.Height; - - if (areaB <= 0) - return 0; - - var minX = Math.Max(boundingBoxA.Left, boundingBoxB.Left); - var minY = Math.Max(boundingBoxA.Top, boundingBoxB.Top); - var maxX = Math.Min(boundingBoxA.Right, boundingBoxB.Right); - var maxY = Math.Min(boundingBoxA.Bottom, boundingBoxB.Bottom); - - var intersectionArea = Math.Max(maxY - minY, 0) * Math.Max(maxX - minX, 0); - - return intersectionArea / (areaA + areaB - intersectionArea); - } } } \ No newline at end of file From a65ae2b3f23047127c5756246942e15ef8624304 Mon Sep 17 00:00:00 2001 From: Luis Quintanilla Date: Wed, 26 Jun 2019 04:23:04 -0400 Subject: [PATCH 5/8] Removing unused variables --- .../ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs index 5cd063d9b..001b0ba16 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs @@ -154,9 +154,6 @@ public IList ParseOutputs(float[] yoloModelOutputs, float thres { var boxes = new List(); - var featuresPerBox = BOX_INFO_FEATURE_COUNT + CLASS_COUNT; - var stride = featuresPerBox * BOXES_PER_CELL; - for (int row = 0; row < ROW_COUNT; row++) { for (int column = 0; column < COL_COUNT; column++) From b5fde5b217b37d1edb82fc8cf333f647591a75c3 Mon Sep 17 00:00:00 2001 From: Luis Quintanilla Date: Fri, 28 Jun 2019 11:43:24 -0400 Subject: [PATCH 6/8] restored old variable names --- .../OnnxModelScorer.cs | 26 +++++++++---------- .../ObjectDetectionConsoleApp/Program.cs | 2 +- ...YoloOutputParser.cs => YoloWinMlParser.cs} | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) rename samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/{YoloOutputParser.cs => YoloWinMlParser.cs} (99%) diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs index 6e5800e7e..7a5e09653 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs @@ -10,17 +10,17 @@ namespace ObjectDetection { class OnnxModelScorer { - private readonly string _imagesFolder; - private readonly string _modelLocation; - private readonly MLContext _mlContext; + private readonly string imagesFolder; + private readonly string modelLocation; + private readonly MLContext mlContext; private IList _boundingBoxes = new List(); public OnnxModelScorer(string imagesFolder, string modelLocation, MLContext mlContext) { - _imagesFolder = imagesFolder; - _modelLocation = modelLocation; - _mlContext = mlContext; + this.imagesFolder = imagesFolder; + this.modelLocation = modelLocation; + this.mlContext = mlContext; } public struct ImageNetSettings @@ -48,12 +48,12 @@ private ITransformer LoadModel(string modelLocation) Console.WriteLine($"Model location: {modelLocation}"); Console.WriteLine($"Default parameters: image size=({ImageNetSettings.imageWidth},{ImageNetSettings.imageHeight})"); - var data = _mlContext.Data.LoadFromEnumerable(new List()); + var data = mlContext.Data.LoadFromEnumerable(new List()); - var pipeline = _mlContext.Transforms.LoadImages(outputColumnName: "image", imageFolder: "", inputColumnName: nameof(ImageNetData.ImagePath)) - .Append(_mlContext.Transforms.ResizeImages(outputColumnName: "image", imageWidth: ImageNetSettings.imageWidth, imageHeight: ImageNetSettings.imageHeight, inputColumnName: "image")) - .Append(_mlContext.Transforms.ExtractPixels(outputColumnName: "image")) - .Append(_mlContext.Transforms.ApplyOnnxModel(modelFile: modelLocation, outputColumnNames: new[] { TinyYoloModelSettings.ModelOutput }, inputColumnNames: new[] { TinyYoloModelSettings.ModelInput })); + var pipeline = mlContext.Transforms.LoadImages(outputColumnName: "image", imageFolder: "", inputColumnName: nameof(ImageNetData.ImagePath)) + .Append(mlContext.Transforms.ResizeImages(outputColumnName: "image", imageWidth: ImageNetSettings.imageWidth, imageHeight: ImageNetSettings.imageHeight, inputColumnName: "image")) + .Append(mlContext.Transforms.ExtractPixels(outputColumnName: "image")) + .Append(mlContext.Transforms.ApplyOnnxModel(modelFile: modelLocation, outputColumnNames: new[] { TinyYoloModelSettings.ModelOutput }, inputColumnNames: new[] { TinyYoloModelSettings.ModelInput })); var model = pipeline.Fit(data); @@ -62,7 +62,7 @@ private ITransformer LoadModel(string modelLocation) private IEnumerable PredictDataUsingModel(IDataView testData, ITransformer model) { - Console.WriteLine($"Images location: {_imagesFolder}"); + Console.WriteLine($"Images location: {imagesFolder}"); Console.WriteLine(""); Console.WriteLine("=====Identify the objects in the images====="); Console.WriteLine(""); @@ -76,7 +76,7 @@ private IEnumerable PredictDataUsingModel(IDataView testData, ITransfor public IEnumerable Score(IDataView data) { - var model = LoadModel(_modelLocation); + var model = LoadModel(modelLocation); return PredictDataUsingModel(data, model); } diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs index d9f745a8f..aac9eb682 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs @@ -36,7 +36,7 @@ public static void Main() IEnumerable probabilities = modelScorer.Score(imageDataView); // Post-process model output - YoloOutputParser parser = new YoloOutputParser(); + YoloWinMlParser parser = new YoloWinMlParser(); var boundingBoxes = probabilities diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloWinMlParser.cs similarity index 99% rename from samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs rename to samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloWinMlParser.cs index 001b0ba16..d18b7000c 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloOutputParser.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/YoloParser/YoloWinMlParser.cs @@ -5,7 +5,7 @@ namespace ObjectDetection.YoloParser { - class YoloOutputParser + class YoloWinMlParser { class CellDimensions : DimensionsBase { } From d12d045f22a820637c9231ffcc6f23888c94fe84 Mon Sep 17 00:00:00 2001 From: Luis Quintanilla Date: Fri, 28 Jun 2019 11:46:01 -0400 Subject: [PATCH 7/8] Updated based on feedback --- .../ObjectDetectionConsoleApp/Program.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs index aac9eb682..996a40261 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/Program.cs @@ -89,10 +89,10 @@ private static void DrawBoundingBox(string inputImageLocation, string outputImag var height = (uint)Math.Min(originalImageHeight - y, box.Dimensions.Height); // Resize To Image - x = (uint)originalImageWidth * x / 416; - y = (uint)originalImageHeight * y / 416; - width = (uint)originalImageWidth * width / 416; - height = (uint)originalImageHeight * height / 416; + x = (uint)originalImageWidth * x / OnnxModelScorer.ImageNetSettings.imageWidth; + y = (uint)originalImageHeight * y / OnnxModelScorer.ImageNetSettings.imageHeight; + width = (uint)originalImageWidth * width / OnnxModelScorer.ImageNetSettings.imageWidth; + height = (uint)originalImageHeight * height / OnnxModelScorer.ImageNetSettings.imageHeight; // Bounding Box Text string text = $"{box.Label} ({(box.Confidence * 100).ToString("0")}%)"; From ba668b71977abbf02ebf3eb3e52422fd65700b84 Mon Sep 17 00:00:00 2001 From: Luis Quintanilla Date: Tue, 2 Jul 2019 15:57:36 -0400 Subject: [PATCH 8/8] Added comments based on feedback --- .../ObjectDetectionConsoleApp/OnnxModelScorer.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs index 7a5e09653..fbd13acc4 100644 --- a/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs +++ b/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx/ObjectDetectionConsoleApp/OnnxModelScorer.cs @@ -48,13 +48,16 @@ private ITransformer LoadModel(string modelLocation) Console.WriteLine($"Model location: {modelLocation}"); Console.WriteLine($"Default parameters: image size=({ImageNetSettings.imageWidth},{ImageNetSettings.imageHeight})"); + // Create IDataView from empty list to obtain input data schema var data = mlContext.Data.LoadFromEnumerable(new List()); + // Define scoring pipeline var pipeline = mlContext.Transforms.LoadImages(outputColumnName: "image", imageFolder: "", inputColumnName: nameof(ImageNetData.ImagePath)) .Append(mlContext.Transforms.ResizeImages(outputColumnName: "image", imageWidth: ImageNetSettings.imageWidth, imageHeight: ImageNetSettings.imageHeight, inputColumnName: "image")) .Append(mlContext.Transforms.ExtractPixels(outputColumnName: "image")) .Append(mlContext.Transforms.ApplyOnnxModel(modelFile: modelLocation, outputColumnNames: new[] { TinyYoloModelSettings.ModelOutput }, inputColumnNames: new[] { TinyYoloModelSettings.ModelInput })); + // Fit scoring pipeline var model = pipeline.Fit(data); return model;