diff --git a/build/Dependencies.props b/build/Dependencies.props index c221bf4f80..375414b61c 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -16,7 +16,7 @@ 3.5.1 2.2.1.1 - 1.1.0 + 0.1.5 0.0.0.7 2.1.3 4.5.0 diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/OnnxTransform.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/OnnxTransform.cs new file mode 100644 index 0000000000..6db1884c00 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/OnnxTransform.cs @@ -0,0 +1,110 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.Runtime.Api; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Transforms; +using System; +using System.Linq; + +namespace Microsoft.ML.Samples.Dynamic +{ + class OnnxTransformExample + { + /// + /// Example use of OnnxEstimator in an ML.NET pipeline + /// + public static void OnnxTransformSample() + { + // Download the squeeznet image model from ONNX model zoo, version 1.2 + // https://github.com/onnx/models/tree/master/squeezenet + var modelPath = @"squeezenet\model.onnx"; + + // Inspect the model's inputs and outputs + var session = new InferenceSession(modelPath); + var inputInfo = session.InputMetadata.First(); + var outputInfo = session.OutputMetadata.First(); + Console.WriteLine($"Input Name is {String.Join(",", inputInfo.Key)}"); + Console.WriteLine($"Input Dimensions are {String.Join(",", inputInfo.Value.Dimensions)}"); + Console.WriteLine($"Output Name is {String.Join(",", outputInfo.Key)}"); + Console.WriteLine($"Output Dimensions are {String.Join(",", outputInfo.Value.Dimensions)}"); + // Results.. + // Input Name is data_0 + // Input Dimensions are 1,3,224,224 + // Output Name is softmaxout_1 + // Output Dimensions are 1,1000,1,1 + + // Create ML pipeline to score the data using OnnxScoringEstimator + var mlContext = new MLContext(); + var data = GetTensorData(); + var idv = mlContext.CreateStreamingDataView(data); + var pipeline = new OnnxScoringEstimator(mlContext, modelPath, new[] { inputInfo.Key }, new[] { outputInfo.Key }); + + // Run the pipeline and get the transformed values + var transformedValues = pipeline.Fit(idv).Transform(idv); + + // Retrieve model scores into Prediction class + var predictions = transformedValues.AsEnumerable(mlContext, reuseRowObject: false); + + // Iterate rows + foreach (var prediction in predictions) + { + int numClasses = 0; + foreach (var classScore in prediction.softmaxout_1.Take(3)) + { + Console.WriteLine($"Class #{numClasses++} score = {classScore}"); + } + Console.WriteLine(new string('-', 10)); + } + + // Results look like below... + // Class #0 score = 4.544065E-05 + // Class #1 score = 0.003845858 + // Class #2 score = 0.0001249467 + // ---------- + // Class #0 score = 4.491953E-05 + // Class #1 score = 0.003848222 + // Class #2 score = 0.0001245592 + // ---------- + } + + /// + /// inputSize is the overall dimensions of the model input tensor. + /// + private const int inputSize = 224 * 224 * 3; + + /// + /// A class to hold sample tensor data. Member name should match + /// the inputs that the model expects (in this case, data_0) + /// + public class TensorData + { + [VectorType(inputSize)] + public float[] data_0 { get; set; } + } + + /// + /// Method to generate sample test data. Returns 2 sample rows. + /// + /// + public static TensorData[] GetTensorData() + { + // This can be any numerical data. Assume image pixel values. + var image1 = Enumerable.Range(0, inputSize).Select(x => (float)x / inputSize).ToArray(); + var image2 = Enumerable.Range(0, inputSize).Select(x => (float)(x + 10000) / inputSize).ToArray(); + return new TensorData[] { new TensorData() { data_0 = image1 }, new TensorData() { data_0 = image2 } }; + } + + /// + /// Class to contain the output values from the transformation. + /// This model generates a vector of 1000 floats. + /// + class Prediction + { + [VectorType(1000)] + public float[] softmaxout_1 { get; set; } + } + } +} \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj index 6b3d9f957b..b938b20b15 100644 --- a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj +++ b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj @@ -13,6 +13,7 @@ + diff --git a/pkg/Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.nupkgproj b/pkg/Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.nupkgproj index 4d64d756fe..b817e809d1 100644 --- a/pkg/Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.nupkgproj +++ b/pkg/Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.nupkgproj @@ -7,7 +7,7 @@ - + diff --git a/src/Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.csproj b/src/Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.csproj index 2298f8d0d7..ce2ac23746 100644 --- a/src/Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.csproj +++ b/src/Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/Microsoft.ML.OnnxTransform/OnnxTransform.cs b/src/Microsoft.ML.OnnxTransform/OnnxTransform.cs index b6eb29286e..e414261336 100644 --- a/src/Microsoft.ML.OnnxTransform/OnnxTransform.cs +++ b/src/Microsoft.ML.OnnxTransform/OnnxTransform.cs @@ -12,13 +12,13 @@ using Microsoft.ML.Runtime.EntryPoints; using Microsoft.ML.Runtime.Internal.Utilities; using Microsoft.ML.Runtime.Model; -using Microsoft.ML.Scoring; +using Microsoft.ML.OnnxRuntime; using Microsoft.ML.Transforms; using Microsoft.ML.StaticPipe; using Microsoft.ML.StaticPipe.Runtime; using Microsoft.ML.Core.Data; using Microsoft.ML.Data; -using OnnxShape = System.Collections.Generic.List; +using OnnxShape = System.Collections.Generic.List; [assembly: LoadableClass(OnnxTransform.Summary, typeof(IDataTransform), typeof(OnnxTransform), typeof(OnnxTransform.Arguments), typeof(SignatureDataTransform), OnnxTransform.UserName, OnnxTransform.ShortName, "OnnxTransform", "OnnxScorer")] @@ -36,6 +36,22 @@ namespace Microsoft.ML.Transforms { + /// + ///

A transform for scoring ONNX models in the ML.NET framework.

+ /// + /// + /// + /// + /// + ///

Supports inferencing of models in 1.2 and 1.3 format, using the + /// Microsoft.ML.OnnxRuntime library + ///

+ ///

The inputs and outputs of the onnx models must of of Tensors. Sequence and Maps are not yet supported.

+ ///

Visit https://github.com/onnx/models to see a list of readily available models to get started with.

+ ///

Refer to http://onnx.ai' for more information about ONNX.

+ /// public sealed class OnnxTransform : RowToRowTransformerBase { public sealed class Arguments : TransformInputBase @@ -75,6 +91,12 @@ private static VersionInfo GetVersionInfo() loaderAssemblyName: typeof(OnnxTransform).Assembly.FullName); } + public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile) + { + var args = new Arguments { ModelFile = modelFile, InputColumns = new string[] { }, OutputColumns = new string[] { } }; + return Create(env, args, input); + } + public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] inputColumns, string[] outputColumns) { var args = new Arguments { ModelFile = modelFile, InputColumns = inputColumns, OutputColumns = outputColumns }; @@ -147,15 +169,15 @@ private OnnxTransform(IHostEnvironment env, Arguments args, byte[] modelBytes = Model = OnnxModel.CreateFromBytes(modelBytes); var modelInfo = Model.ModelInfo; - Inputs = args.InputColumns; - Outputs = args.OutputColumns; - OutputTypes = new ColumnType[args.OutputColumns.Length]; + Inputs = (args.InputColumns.Count() == 0 ) ? Model.InputNames.ToArray() : args.InputColumns; + Outputs = (args.OutputColumns.Count() == 0 ) ? Model.OutputNames.ToArray() : args.OutputColumns; + OutputTypes = new ColumnType[Outputs.Length]; var numModelOutputs = Model.ModelInfo.OutputsInfo.Length; - for (int i=0; i < args.OutputColumns.Length; i++) + for (int i=0; i < Outputs.Length; i++) { - var idx = Model.OutputNames.IndexOf(args.OutputColumns[i]); + var idx = Model.OutputNames.IndexOf(Outputs[i]); if (idx < 0) - throw Host.Except($"Column {args.OutputColumns[i]} doesn't match output node names of model"); + throw Host.Except($"Column {Outputs[i]} doesn't match output node names of model"); var outputNodeInfo = Model.ModelInfo.OutputsInfo[idx]; var shape = outputNodeInfo.Shape; @@ -165,6 +187,11 @@ private OnnxTransform(IHostEnvironment env, Arguments args, byte[] modelBytes = _args = args; } + public OnnxTransform(IHostEnvironment env, string modelFile) + : this(env, new Arguments() { ModelFile = modelFile, InputColumns = new string[] { }, OutputColumns = new string[] { } }) + { + } + public OnnxTransform(IHostEnvironment env, string modelFile, string inputColumn, string outputColumn) : this(env, new Arguments() { ModelFile = modelFile, InputColumns = new[] { inputColumn }, OutputColumns = new[] { outputColumn } }) { @@ -223,7 +250,7 @@ private sealed class Mapper : MapperBase private readonly int[] _inputColIndices; private readonly bool[] _isInputVector; private readonly OnnxShape[] _inputTensorShapes; - private readonly DataType[] _inputOnnxTypes; + private readonly System.Type[] _inputOnnxTypes; public Mapper(OnnxTransform parent, Schema inputSchema) : base(Contracts.CheckRef(parent, nameof(parent)).Host.Register(nameof(Mapper)), inputSchema) @@ -233,7 +260,7 @@ public Mapper(OnnxTransform parent, Schema inputSchema) : _inputColIndices = new int[_parent.Inputs.Length]; _isInputVector = new bool[_parent.Inputs.Length]; _inputTensorShapes = new OnnxShape[_parent.Inputs.Length]; - _inputOnnxTypes = new DataType[_parent.Inputs.Length]; + _inputOnnxTypes = new System.Type[_parent.Inputs.Length]; var model = _parent.Model; for (int i = 0; i < _parent.Inputs.Length; i++) @@ -289,36 +316,39 @@ public override Func GetDependencies(Func activeOutput) public override void Save(ModelSaveContext ctx) => _parent.Save(ctx); - private interface ITensorValueGetter + private interface INamedOnnxValueGetter { - Tensor GetTensor(); + NamedOnnxValue GetNamedOnnxValue(); } private class OutputCache { public long Position; - public Dictionary Outputs; + public Dictionary Outputs; public OutputCache() { Position = -1; - Outputs = new Dictionary(); + Outputs = new Dictionary(); } } - private void UpdateCacheIfNeeded(long position, ITensorValueGetter[] srcTensorGetters, string[] activeOutputColNames, OutputCache outputCache) + private void UpdateCacheIfNeeded(long position, INamedOnnxValueGetter[] srcNamedOnnxValueGetters, string[] activeOutputColNames, OutputCache outputCache) { if (outputCache.Position != position) { - var inputTensors = new List(); + var inputNameOnnxValues = new List(); for (int i = 0; i < _inputColIndices.Length; i++) - inputTensors.Add(srcTensorGetters[i].GetTensor()); - - var outputTensors = _parent.Model.Run(inputTensors); - Contracts.Assert(outputTensors.Count > 0); + { + inputNameOnnxValues.Add(srcNamedOnnxValueGetters[i].GetNamedOnnxValue()); + } - for (int j = 0; j < outputTensors.Count; j++) - outputCache.Outputs[activeOutputColNames[j]] = outputTensors[j]; + var outputNamedOnnxValues = _parent.Model.Run(inputNameOnnxValues); + Contracts.Assert(outputNamedOnnxValues.Count > 0); + foreach (var outputNameOnnxValue in outputNamedOnnxValues) + { + outputCache.Outputs[outputNameOnnxValue.Name] = outputNameOnnxValue; + } outputCache.Position = position; } } @@ -333,93 +363,110 @@ protected override Delegate MakeGetter(IRow input, int iinfo, Func ac var activeOutputColNames = _parent.Outputs.Where((x, i) => activeOutput(i)).ToArray(); var type = OnnxUtils.OnnxToMlNetType(_parent.Model.ModelInfo.OutputsInfo[iinfo].Type).RawType; Host.Assert(type == _parent.OutputTypes[iinfo].ItemType.RawType); - var srcTensorGetters = GetTensorValueGetters(input, _inputColIndices, _isInputVector, _inputOnnxTypes, _inputTensorShapes); - return Utils.MarshalInvoke(MakeGetter, type, input, iinfo, srcTensorGetters, activeOutputColNames, outputCache); + var srcNamedValueGetters = GetNamedOnnxValueGetters(input, _parent.Inputs, _inputColIndices, _isInputVector, _inputOnnxTypes, _inputTensorShapes); + return Utils.MarshalInvoke(MakeGetter, type, input, iinfo, srcNamedValueGetters, activeOutputColNames, outputCache); } - private Delegate MakeGetter(IRow input, int iinfo, ITensorValueGetter[] srcTensorGetters, string[] activeOutputColNames, OutputCache outputCache) + private Delegate MakeGetter(IRow input, int iinfo, INamedOnnxValueGetter[] srcNamedValueGetters, string[] activeOutputColNames, OutputCache outputCache) { Host.AssertValue(input); ValueGetter> valuegetter = (ref VBuffer dst) => { - UpdateCacheIfNeeded(input.Position, srcTensorGetters, activeOutputColNames, outputCache); - var tensor = outputCache.Outputs[_parent.Outputs[iinfo]]; - var editor = VBufferEditor.Create(ref dst, tensor.GetSize()); - OnnxUtils.CopyTo(tensor, editor.Values); + UpdateCacheIfNeeded(input.Position, srcNamedValueGetters, activeOutputColNames, outputCache); + var namedOnnxValue = outputCache.Outputs[_parent.Outputs[iinfo]]; + var denseTensor = namedOnnxValue.AsTensor() as System.Numerics.Tensors.DenseTensor; + if (denseTensor == null) + throw Host.Except($"Output column {namedOnnxValue.Name} doesn't contain a DenseTensor of expected type {typeof(T)}"); + var editor = VBufferEditor.Create(ref dst, (int) denseTensor.Length); + denseTensor.Buffer.Span.CopyTo(editor.Values); dst = editor.Commit(); }; return valuegetter; } - private static ITensorValueGetter[] GetTensorValueGetters(IRow input, + private static INamedOnnxValueGetter[] GetNamedOnnxValueGetters(IRow input, + string[] inputColNames, int[] inputColIndices, bool[] isInputVector, - DataType[] onnxInputTypes, + System.Type[] onnxInputTypes, OnnxShape[] onnxInputShapes) { - var srcTensorGetters = new ITensorValueGetter[inputColIndices.Length]; + var srcNamedOnnxValueGetters = new INamedOnnxValueGetter[inputColIndices.Length]; for (int i = 0; i < inputColIndices.Length; i++) { int colIndex = inputColIndices[i]; - srcTensorGetters[i] = CreateTensorValueGetter(input, onnxInputTypes[i], isInputVector[i], colIndex, onnxInputShapes[i]); + srcNamedOnnxValueGetters[i] = CreateNamedOnnxValueGetter(input, onnxInputTypes[i], isInputVector[i], inputColNames[i], colIndex, onnxInputShapes[i]); } - return srcTensorGetters; + return srcNamedOnnxValueGetters; } - private static ITensorValueGetter CreateTensorValueGetter(IRow input, DataType onnxType, bool isVector, int colIndex, OnnxShape onnxShape) + private static INamedOnnxValueGetter CreateNamedOnnxValueGetter(IRow input, System.Type onnxType, bool isVector, string colName, int colIndex, OnnxShape onnxShape) { var type = OnnxUtils.OnnxToMlNetType(onnxType).RawType; Contracts.AssertValue(type); - return Utils.MarshalInvoke(CreateTensorValueGetter, type, input, isVector, colIndex, onnxShape); + return Utils.MarshalInvoke(CreateNameOnnxValueGetter, type, input, isVector, colName, colIndex, onnxShape); } - private static ITensorValueGetter CreateTensorValueGetter(IRow input, bool isVector, int colIndex, OnnxShape onnxShape) + private static INamedOnnxValueGetter CreateNameOnnxValueGetter(IRow input, bool isVector, string colName, int colIndex, OnnxShape onnxShape) { if (isVector) - return new TensorValueGetterVec(input, colIndex, onnxShape); - return new TensorValueGetter(input, colIndex); + return new NamedOnnxValueGetterVec(input, colName, colIndex, onnxShape); + return new NameOnnxValueGetter(input, colName, colIndex); } - private class TensorValueGetter : ITensorValueGetter + private class NameOnnxValueGetter : INamedOnnxValueGetter { private readonly ValueGetter _srcgetter; + private readonly string _colName; - public TensorValueGetter(IRow input, int colIndex) + public NameOnnxValueGetter(IRow input, string colName, int colIndex) { + _colName = colName; _srcgetter = input.GetGetter(colIndex); } - public Tensor GetTensor() + public NamedOnnxValue GetNamedOnnxValue() { var scalar = default(T); _srcgetter(ref scalar); - return OnnxUtils.CreateScalarTensor(scalar); + return OnnxUtils.CreateScalarNamedOnnxValue(_colName, scalar); } } - private class TensorValueGetterVec : ITensorValueGetter + private class NamedOnnxValueGetterVec : INamedOnnxValueGetter { private readonly ValueGetter> _srcgetter; private readonly OnnxShape _tensorShape; + private readonly string _colName; private VBuffer _vBuffer; private VBuffer _vBufferDense; - public TensorValueGetterVec(IRow input, int colIndex, OnnxShape tensorShape) + public NamedOnnxValueGetterVec(IRow input, string colName, int colIndex, OnnxShape tensorShape) { _srcgetter = input.GetGetter>(colIndex); _tensorShape = tensorShape; + _colName = colName; _vBuffer = default; _vBufferDense = default; } - public Tensor GetTensor() + public NamedOnnxValue GetNamedOnnxValue() { _srcgetter(ref _vBuffer); _vBuffer.CopyToDense(ref _vBufferDense); - return OnnxUtils.CreateTensor(_vBufferDense.GetValues(), _tensorShape); + return OnnxUtils.CreateNamedOnnxValue(_colName, _vBufferDense.GetValues(), _tensorShape); } } } } + + /// + /// A class implementing the estimator interface of the OnnxTransform. + /// public sealed class OnnxScoringEstimator : TrivialEstimator { + public OnnxScoringEstimator(IHostEnvironment env, string modelFile) + : this(env, new OnnxTransform(env, modelFile, new string[] { }, new string[] { })) + { + } + public OnnxScoringEstimator(IHostEnvironment env, string modelFile, string[] inputs, string[] outputs) : this(env, new OnnxTransform(env, modelFile, inputs, outputs)) { @@ -459,7 +506,7 @@ public override SchemaShape GetOutputSchema(SchemaShape inputSchema) { resultDic[Transformer.Outputs[i]] = new SchemaShape.Column(Transformer.Outputs[i], Transformer.OutputTypes[i].IsKnownSizeVector ? SchemaShape.Column.VectorKind.Vector - : SchemaShape.Column.VectorKind.VariableVector, NumberType.R4, false); + : SchemaShape.Column.VectorKind.VariableVector, Transformer.OutputTypes[i].ItemType, false); } return new SchemaShape(resultDic.Values); } diff --git a/src/Microsoft.ML.OnnxTransform/OnnxUtils.cs b/src/Microsoft.ML.OnnxTransform/OnnxUtils.cs index 611359d9cc..001f2ce9d8 100644 --- a/src/Microsoft.ML.OnnxTransform/OnnxUtils.cs +++ b/src/Microsoft.ML.OnnxTransform/OnnxUtils.cs @@ -4,40 +4,39 @@ using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.Data; -using Microsoft.ML.Runtime.Internal.Utilities; -using Microsoft.ML.Scoring; +using Microsoft.ML.OnnxRuntime; +using System.Numerics.Tensors; using System; using System.Collections.Generic; using System.IO; using System.Linq; -using Microsoft.ML.StaticPipe; -using OnnxShape = System.Collections.Generic.List; -using Microsoft.ML.Data; +using OnnxShape = System.Collections.Generic.List; namespace Microsoft.ML.Transforms { /// - /// OnnxModel is a facad for ModelManager. ModelManager is provided by Sonoma API, - /// and it has a lot of functionality (multiple models, multiple versions) that are not - /// needed by Onnx transform, which only needs a single model. This facad simplifies the - /// usage of onnx model. + /// OnnxModel is a utility class to load ONNX models and retrieve metadata + /// for inputs and outputs. The metadata includes the names, shapes and types + /// It provides API to open a session, score tensors (NamedOnnxValues) and return + /// the results. /// internal sealed class OnnxModel { + /// /// OnnxModelInfo contains the data that we should get from - /// Sonoma API once that functionality is added. + /// OnnxRuntime API once that functionality is added. /// public sealed class OnnxModelInfo { public readonly OnnxNodeInfo[] InputsInfo; public readonly OnnxNodeInfo[] OutputsInfo; - public OnnxModelInfo(OnnxNodeInfo[] inputsInfo, OnnxNodeInfo[] outputsInfo) + public OnnxModelInfo(IEnumerable inputsInfo, IEnumerable outputsInfo) { - InputsInfo = inputsInfo; - OutputsInfo = outputsInfo; + InputsInfo = inputsInfo.ToArray(); + OutputsInfo = outputsInfo.ToArray(); } } @@ -47,11 +46,20 @@ public OnnxModelInfo(OnnxNodeInfo[] inputsInfo, OnnxNodeInfo[] outputsInfo) /// public class OnnxNodeInfo { + /// + /// The Name of the node + /// public readonly string Name; + /// + /// The shape of the node + /// public readonly OnnxShape Shape; - public readonly DataType Type; + /// + /// The type of the node + /// + public readonly System.Type Type; - public OnnxNodeInfo(string name, OnnxShape shape, DataType type) + public OnnxNodeInfo(string name, OnnxShape shape, System.Type type) { Name = name; Shape = shape; @@ -60,29 +68,25 @@ public OnnxNodeInfo(string name, OnnxShape shape, DataType type) } public readonly OnnxModelInfo ModelInfo; - - private static readonly int _ignoredVersion = int.MaxValue; - private readonly ModelManager _modelManager; + private readonly InferenceSession _session; private readonly string _modelFile; - private readonly string _modelName; public readonly List InputNames; public readonly List OutputNames; public OnnxModel(string modelFile) { _modelFile = modelFile; - - // Load the onnx model - var modelFileInfo = new FileInfo(modelFile); - _modelName = Path.GetFileNameWithoutExtension(modelFileInfo.Name); - _modelManager = new ModelManager(modelFileInfo.Directory.FullName, true); - _modelManager.InitOnnxModel(_modelName, _ignoredVersion); - + _session = new InferenceSession(modelFile); ModelInfo = new OnnxModelInfo(GetInputsInfo(), GetOutputsInfo()); InputNames = ModelInfo.InputsInfo.Select(i => i.Name).ToList(); OutputNames = ModelInfo.OutputsInfo.Select(i => i.Name).ToList(); } + /// + /// Create an OnnxModel from a byte[] + /// + /// + /// OnnxModel public static OnnxModel CreateFromBytes(byte[] modelBytes) { var tempModelDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); @@ -98,239 +102,114 @@ public static OnnxModel CreateFromBytes(byte[] modelBytes) // or keep the dir/file and write proper cleanup when application closes } - public List Run(List inputTensors) + /// + /// Uses an open session to score a list of NamedOnnxValues. + /// + /// The NamedOnnxValues to score + /// Resulting output NamedOnnxValues list + public IReadOnlyCollection Run(List inputNamedOnnxValues) { - var outputTensors = _modelManager.RunModel( - _modelName, _ignoredVersion, InputNames, inputTensors, OutputNames); - - return outputTensors; + return _session.Run(inputNamedOnnxValues); } + /// + /// Convert the model to a byte array. + /// + /// byte[] public byte[] ToByteArray() { return File.ReadAllBytes(_modelFile); } - private OnnxNodeInfo[] GetInputsInfo() - { - return DictToNodesInfo( - _modelManager.GetInputTypeDict(_modelName, _ignoredVersion), - _modelManager.GetInputShapesDict(_modelName, _ignoredVersion)); - } - - private OnnxNodeInfo[] GetOutputsInfo() + /// + /// Returns input metadata of the ONNX model. + /// + /// OnnxNodeInfo[] + private IEnumerable GetInputsInfo() { - return DictToNodesInfo( - _modelManager.GetOutputTypeDict(_modelName, _ignoredVersion), - _modelManager.GetOutputShapesDict(_modelName, _ignoredVersion)); + return _session.InputMetadata.Select(kv => new OnnxNodeInfo(kv.Key, kv.Value.Dimensions.ToList(), kv.Value.ElementType)); } - private static OnnxNodeInfo[] DictToNodesInfo( - Dictionary typeDict, - Dictionary shapeDictArray) + /// + /// Returns output metadata of the ONNX model. + /// + /// + private IEnumerable GetOutputsInfo() { - var shapeDict = new Dictionary>(); - foreach (var key in shapeDictArray.Keys) - shapeDict.Add(key, shapeDictArray[key].ToList()); - - var sameKey = typeDict.Count == shapeDict.Count && - typeDict.Keys.SequenceEqual(shapeDict.Keys); - Contracts.Assert(sameKey, "Type and shape dictionaries should have the same keys"); - return typeDict.Select(kv => new OnnxNodeInfo( - name: kv.Key, type: kv.Value, shape: shapeDict[kv.Key])).OrderBy(x => x.Name).ToArray(); + return _session.OutputMetadata.Select(kv => new OnnxNodeInfo(kv.Key, kv.Value.Dimensions.ToList(), kv.Value.ElementType)); } } internal sealed class OnnxUtils { - /// - /// Sonoma API only provides Tensor() constructors with overloaded - /// versions based on data type. - /// - - private static Dictionary _typeMap; - - public static Tensor CreateScalarTensor(T data) - { - if (typeof(T) == typeof(System.Boolean)) - { - return new Tensor((System.Boolean)(object)data); - } - else if (typeof(T) == typeof(System.Byte)) - { - return new Tensor((System.Byte)(object)data); - } - else if (typeof(T) == typeof(System.Char)) - { - return new Tensor((System.Char)(object)data); - } - else if (typeof(T) == typeof(System.Double)) - { - return new Tensor((System.Double)(object)data); - } - else if (typeof(T) == typeof(System.Single)) - { - return new Tensor((System.Single)(object)data); - } - else if (typeof(T) == typeof(System.Int32)) - { - return new Tensor((System.Int32)(object)data); - } - else if (typeof(T) == typeof(System.Int64)) - { - return new Tensor((System.Int64)(object)data); - } - else if (typeof(T) == typeof(System.SByte)) - { - return new Tensor((System.SByte)(object)data); - } - else if (typeof(T) == typeof(System.Int16)) - { - return new Tensor((System.Int16)(object)data); - } - else if (typeof(T) == typeof(System.UInt32)) - { - return new Tensor((System.UInt32)(object)data); - } - else if (typeof(T) == typeof(System.UInt64)) - { - return new Tensor((System.UInt64)(object)data); - } - else if (typeof(T) == typeof(System.UInt16)) - { - return new Tensor((System.UInt16)(object)data); - } - throw new NotSupportedException($"Unsupported type {typeof(T)}"); - } + private static HashSet _onnxTypeMap = + new HashSet + { + typeof(Double), + typeof(Single), + typeof(Int16), + typeof(Int32), + typeof(Int64), + typeof(UInt16), + typeof(UInt32), + typeof(UInt64) + }; + private static Dictionary _typeToKindMap= + new Dictionary + { + { typeof(Single) , DataKind.R4}, + { typeof(Double) , DataKind.R8}, + { typeof(Int16) , DataKind.I2}, + { typeof(Int32) , DataKind.I4}, + { typeof(Int64) , DataKind.I8}, + { typeof(UInt16) , DataKind.U2}, + { typeof(UInt32) , DataKind.U4}, + { typeof(UInt64) , DataKind.U8}, + { typeof(String) , DataKind.TX}, + { typeof(Boolean) , DataKind.BL}, + }; /// - /// Sonoma API only provides Tensor() constructors with overloaded versions - /// based on data type. ML.NET cannot use the overloaded version and requires - /// generic version. CreateTensor<T> is generic wrapper on top of - /// overloaded Tensor(T[] data, OnnxShape shape) constructors. + /// Creates a NamedOnnxValue from a scalar value. /// - public static Tensor CreateTensor(ReadOnlySpan data, OnnxShape shape) + /// The type of the Tensor contained in the NamedOnnxValue + /// The name of the NamedOnnxValue + /// The data values of the Tensor + /// NamedOnnxValue + public static NamedOnnxValue CreateScalarNamedOnnxValue(string name, T data) { - if (typeof(T) == typeof(System.Boolean)) - { - return new Tensor((System.Boolean[])(object)data.ToArray(), shape.ToArray()); - } - else if (typeof(T) == typeof(System.Double)) - { - return new Tensor((System.Double[])(object)data.ToArray(), shape.ToArray()); - } - else if (typeof(T) == typeof(System.Single)) - { - return new Tensor((System.Single[])(object)data.ToArray(), shape.ToArray()); - } - else if (typeof(T) == typeof(System.Int32)) - { - return new Tensor((System.Int32[])(object)data.ToArray(), shape.ToArray()); - } - else if (typeof(T) == typeof(System.Int64)) - { - return new Tensor((System.Int64[])(object)data.ToArray(), shape.ToArray()); - } - throw new NotImplementedException($"Not implemented type {typeof(T)}"); + if (!_onnxTypeMap.Contains(typeof(T))) + throw new NotImplementedException($"Not implemented type {typeof(T)}"); + return NamedOnnxValue.CreateFromTensor(name, new DenseTensor(new T[] { data }, new int[] { 1 })); } /// - /// Sonoma API only provides CopyTo() functions with overloaded versions - /// based on data type. ML.NET cannot use the overloaded version and requires - /// generic version. CopyTo<T> is generic wrapper on top of - /// overloaded Tensor.CopyTo(List<T> dst) methods. - /// Also Tensor.CopyTo(List<T> dst) requires a list input, whereas ML.NET - /// provides array buffers to copy values to. This mismatch causes an extra copy. + /// Create a NamedOnnxValue from vbuffer span. Checks if the tensor type + /// is supported by OnnxRuntime prior to execution. /// - public static unsafe void CopyTo(Tensor tensor, Span dst) + /// The type of the Tensor contained in the NamedOnnxValue + /// The name of the NamedOnnxValue + /// A span containing the data + /// The shape of the Tensor being created + /// NamedOnnxValue + public static NamedOnnxValue CreateNamedOnnxValue(string name, ReadOnlySpan data, OnnxShape shape) { - var typeMap = SystemTypeToOnnxType(); - if (typeMap.ContainsKey(typeof(T))) - { - if (tensor.GetDataType() != typeMap[typeof(T)]) - { - throw new InvalidOperationException( string.Format("Cannot copy source tensor of type {0} to managed type {1}.", tensor.GetDataType(), typeof(T))); - } - Span tensorSpan = new Span(tensor.UnsafeGetData().ToPointer(), tensor.GetSize()); - tensorSpan.CopyTo(dst); - // TODO: the CopyTo() function is susceptible to GC reclaiming tensor - // during the method call. Use KeepAlive for now, and remove - // after permanent fix in CopyTo(). - } - else + if (!_onnxTypeMap.Contains(typeof(T))) throw new NotImplementedException($"Not implemented type {typeof(T)}"); - GC.KeepAlive(tensor); + return NamedOnnxValue.CreateFromTensor(name, new DenseTensor(data.ToArray(), shape.Select(x => (int)x).ToArray())); } - public static PrimitiveType OnnxToMlNetType(DataType type) - { - DataKind kind; - switch (type) - { - case DataType.Type_Float: - kind = DataKind.R4; - break; - - case DataType.Type_Double: - kind = DataKind.R8; - break; - - case DataType.Type_Int8: - kind = DataKind.I1; - break; - - case DataType.Type_Int16: - kind = DataKind.I2; - break; - - case DataType.Type_Int32: - kind = DataKind.I4; - break; - - case DataType.Type_Int64: - kind = DataKind.I8; - break; - - case DataType.Type_Uint8: - kind = DataKind.U1; - break; - - case DataType.Type_Uint16: - kind = DataKind.U2; - break; - - case DataType.Type_String: - kind = DataKind.TX; - break; - - case DataType.Type_Bool: - kind = DataKind.BL; - break; - - case DataType.Type_Invalid: - default: - throw Contracts.ExceptNotSupp("Onnx type not supported", type); - } - - return PrimitiveType.FromKind(kind); - } - - internal static Dictionary SystemTypeToOnnxType() + /// + /// Converts a Onnx type, that follows the System.Type convention + /// to the type system ML.NET recognizes (e.g. I4, I8, R4 etc.) + /// + /// + /// + public static PrimitiveType OnnxToMlNetType(System.Type type) { - if (_typeMap == null) - { - _typeMap = new Dictionary - { - { typeof(Boolean) , DataType.Type_Bool }, - { typeof(Double) , DataType.Type_Double }, - { typeof(Single) , DataType.Type_Float }, - { typeof(Int16) , DataType.Type_Int16 }, - { typeof(Int32) , DataType.Type_Int32 }, - { typeof(Int64) , DataType.Type_Int64 }, - { typeof(UInt16) , DataType.Type_Uint16 } - }; - } - return _typeMap; + if (!_typeToKindMap.ContainsKey(type)) + throw Contracts.ExceptNotSupp("Onnx type not supported", type); + return PrimitiveType.FromKind(_typeToKindMap[type]); } } } diff --git a/test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs b/test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs index af6a26d440..cf60bf28d3 100644 --- a/test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs +++ b/test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs @@ -13,6 +13,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Runtime.InteropServices; using Xunit; using Xunit.Abstractions; @@ -298,8 +299,10 @@ public void OnnxModelMultiInput() { getScoresa(ref buffera); getScoresb(ref bufferb); - Console.WriteLine(buffera.GetValues().ToArray()); Assert.Equal(5, buffera.Length); + Assert.Equal(5, bufferb.Length); + Assert.Equal(0, buffera.GetValues().ToArray().Sum()); + Assert.Equal(30, bufferb.GetValues().ToArray().Sum()); } } }