From 2c7a62b21bef0326bf183e8c6b8db667ada07959 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Tue, 10 Mar 2020 09:24:42 +0800 Subject: [PATCH 01/36] add root cause localization transformer --- .../TimeSeries/LocalizeRootCauseByDT.cs | 76 +++ .../DTRootCauseLocalization.cs | 490 ++++++++++++++++++ .../ExtensionsCatalog.cs | 17 + .../TimeSeriesDirectApi.cs | 1 + 4 files changed, 584 insertions(+) create mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseByDT.cs create mode 100644 src/Microsoft.ML.TimeSeries/DTRootCauseLocalization.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseByDT.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseByDT.cs new file mode 100644 index 0000000000..94d117fd45 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseByDT.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.IO; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Transforms.TimeSeries; + +namespace Samples.Dynamic +{ + public static class LocalizeRootCause + { + public static void Example() + { + // Create a new ML context, for ML.NET operations. It can be used for + // exception tracking and logging, as well as the source of randomness. + var mlContext = new MLContext(); + + // Create an empty list as the dataset. The 'NormalizeText' API does not + // require training data as the estimator ('TextNormalizingEstimator') + // created by 'NormalizeText' API is not a trainable estimator. The + // empty list is only needed to pass input schema to the pipeline. + var emptySamples = new List(); + + // Convert sample list to an empty IDataView. + var emptyDataView = mlContext.Data.LoadFromEnumerable(emptySamples); + + // A pipeline for localizeing root cause. + var localizePipeline = mlContext.Transforms.LocalizeRootCauseByDT(nameof(RootCauseLocalizationTransformedData.RootCause), nameof(RootCauseLocalizationData.Input)); + + // Fit to data. + var localizeTransformer = localizePipeline.Fit(emptyDataView); + + // Create the prediction engine to get the root cause result from the + // input data. + var predictionEngine = mlContext.Model.CreatePredictionEngine(localizeTransformer); + + // Call the prediction API. + var data = new RootCauseLocalizationData(new DateTime(), new Dictionary(), new List() { new MetricSlice(new DateTime(), new List()) }, "SUM", "SUM"); + + var prediction = predictionEngine.Predict(data); + + // Print the localization result. + Console.WriteLine($"Localized result: {prediction.RootCause}"); + } + + + private class RootCauseLocalizationData + { + [RootCauseLocalizationInputType] + public RootCauseLocalizationInput Input { get; set; } + + public RootCauseLocalizationData() + { + Input = null; + } + + public RootCauseLocalizationData(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices,String aggregateType, string aggregateSymbol) + { + Input = new RootCauseLocalizationInput(anomalyTimestamp, anomalyDimensions, slices, DTRootCauseLocalizationEstimator.AggregateType.Sum, aggregateSymbol); + } + } + + private class RootCauseLocalizationTransformedData + { + [RootCauseType()] + public RootCause RootCause { get; set; } + + public RootCauseLocalizationTransformedData() + { + RootCause = null; + } + } + } +} diff --git a/src/Microsoft.ML.TimeSeries/DTRootCauseLocalization.cs b/src/Microsoft.ML.TimeSeries/DTRootCauseLocalization.cs new file mode 100644 index 0000000000..c70dd398ff --- /dev/null +++ b/src/Microsoft.ML.TimeSeries/DTRootCauseLocalization.cs @@ -0,0 +1,490 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.ML; +using Microsoft.ML.CommandLine; +using Microsoft.ML.Data; +using Microsoft.ML.Internal.Utilities; +using Microsoft.ML.Runtime; +using Microsoft.ML.Transforms.TimeSeries; + +[assembly: LoadableClass(DTRootCauseLocalizationTransformer.Summary, typeof(IDataTransform), typeof(DTRootCauseLocalizationTransformer), typeof(DTRootCauseLocalizationTransformer.Options), typeof(SignatureDataTransform), + DTRootCauseLocalizationTransformer.UserName, "DTRootCauseLocalizationTransform", "DTRootCauseLocalization")] + +[assembly: LoadableClass(DTRootCauseLocalizationTransformer.Summary, typeof(IDataTransform), typeof(DTRootCauseLocalizationTransformer), null, typeof(SignatureLoadDataTransform), + DTRootCauseLocalizationTransformer.UserName, DTRootCauseLocalizationTransformer.LoaderSignature)] + +[assembly: LoadableClass(typeof(DTRootCauseLocalizationTransformer), null, typeof(SignatureLoadModel), + DTRootCauseLocalizationTransformer.UserName, DTRootCauseLocalizationTransformer.LoaderSignature)] + +[assembly: LoadableClass(typeof(IRowMapper), typeof(DTRootCauseLocalizationTransformer), null, typeof(SignatureLoadRowMapper), + DTRootCauseLocalizationTransformer.UserName, DTRootCauseLocalizationTransformer.LoaderSignature)] + +namespace Microsoft.ML.Transforms.TimeSeries +{ + public sealed class RootCauseLocalizationInputTypeAttribute : DataViewTypeAttribute + { + /// + /// Create a root cause localizagin input type. + /// + public RootCauseLocalizationInputTypeAttribute() + { + } + + /// + /// Equal function. + /// + public override bool Equals(DataViewTypeAttribute other) + { + if (!(other is RootCauseLocalizationInputTypeAttribute otherAttribute)) + return false; + return true; + } + + /// + /// Produce the same hash code for all RootCauseLocalizationInputTypeAttribute. + /// + public override int GetHashCode() + { + return 0; + } + + public override void Register() + { + DataViewTypeManager.Register(new RootCauseLocalizationInputDataViewType(), typeof(RootCauseLocalizationInput), this); + } + } + + public sealed class RootCauseTypeAttribute : DataViewTypeAttribute + { + /// + /// Create an root cause type. + /// + public RootCauseTypeAttribute() + { + } + + /// + /// RootCauseTypeAttribute with the same type should equal. + /// + public override bool Equals(DataViewTypeAttribute other) + { + if (other is RootCauseTypeAttribute otherAttribute) + return true; + return false; + } + + /// + /// Produce the same hash code for all RootCauseTypeAttribute. + /// + public override int GetHashCode() + { + return 0; + } + + public override void Register() + { + DataViewTypeManager.Register(new RootCauseDataViewType(), typeof(RootCause), this); + } + } + + public sealed class RootCause + { + public List Items { get; set; } + } + + public sealed class RootCauseItems { + public double Score; + public List Path; + public Dictionary RootCause; + public AnomalyDirection Direction; + } + + public enum AnomalyDirection { + /// + /// the value is larger than expected value. + /// + Up = 0, + /// + /// the value is lower than expected value. + /// + Down = 1 + } + + public sealed class RootCauseLocalizationInput + { + public DateTime AnomalyTimestamp { get; set; } + + public Dictionary AnomalyDimensions { get; set; } + + public List Slices { get; set; } + + public DTRootCauseLocalizationEstimator.AggregateType AggType{ get; set; } + + public string AggSymbol { get; set; } + + public RootCauseLocalizationInput(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, DTRootCauseLocalizationEstimator.AggregateType aggregateType, string aggregateSymbol) { + AnomalyTimestamp = anomalyTimestamp; + AnomalyDimensions = anomalyDimensions; + Slices = slices; + AggType = aggregateType; + AggSymbol = aggregateSymbol; + } + public void Dispose() + { + AnomalyDimensions = null; + Slices = null; + } + } + + public sealed class MetricSlice + { + public DateTime TimeStamp { get; set; } + public List Points { get; set; } + + public MetricSlice(DateTime timeStamp, List points) { + TimeStamp = timeStamp; + Points = points; + } + } + + public sealed class Point { + public double Value { get; set; } + public double ExpectedValue { get; set; } + public bool IsAnomaly { get; set; } + public Dictionary Dimensions{ get; set; } + } + + public sealed class RootCauseDataViewType : StructuredDataViewType + { + public RootCauseDataViewType() + : base(typeof(RootCause)) + { + } + + public override bool Equals(DataViewType other) + { + if (other == this) + return true; + if (!(other is RootCauseDataViewType tmp)) + return false; + return true; + } + + public override int GetHashCode() + { + return 0; + } + + public override string ToString() + { + return typeof(RootCauseDataViewType).Name; + } + } + + public sealed class RootCauseLocalizationInputDataViewType : StructuredDataViewType + { + public RootCauseLocalizationInputDataViewType() + : base(typeof(RootCauseLocalizationInput)) + { + } + + public override bool Equals(DataViewType other) + { + if (!(other is RootCauseLocalizationInputDataViewType tmp)) + return false; + return true; + } + + public override int GetHashCode() + { + return 0; + } + + public override string ToString() + { + return typeof(RootCauseLocalizationInputDataViewType).Name; + } + } + + // REVIEW: Rewrite as LambdaTransform to simplify. + // REVIEW: Should it be separate transform or part of ImageResizerTransform? + /// + /// resulting from fitting an . + /// + public sealed class DTRootCauseLocalizationTransformer : OneToOneTransformerBase + { + internal sealed class Column : OneToOneColumn + { + internal static Column Parse(string str) + { + var res = new Column(); + if (res.TryParse(str)) + return res; + return null; + } + + internal bool TryUnparse(StringBuilder sb) + { + Contracts.AssertValue(sb); + return TryUnparseCore(sb); + } + } + + internal class Options : TransformInputBase + { + [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", Name = "Column", ShortName = "col", SortOrder = 1)] + public Column[] Columns; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Weight for getting the score for the root cause item.", ShortName = "Beta", SortOrder = 2)] + public double Beta = DTRootCauseLocalizationEstimator.Defaults.Beta; + + } + + internal const string Summary = "Localize root cause for anomaly."; + + internal const string UserName = "DT Root Cause Localization Transform"; + internal const string LoaderSignature = "DTRootCauseLTransform"; + + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "DTRCL", + verWrittenCur: 0x00010001, // Initial + verReadableCur: 0x00010001, + verWeCanReadBack: 0x00010001, + loaderSignature: LoaderSignature, + loaderAssemblyName: typeof(DTRootCauseLocalizationTransformer).Assembly.FullName); + } + + private const string RegistrationName = "RootCauseLocalization"; + + /// + /// The input and output column pairs passed to this . + /// + internal IReadOnlyCollection<(string outputColumnName, string inputColumnName)> Columns => ColumnPairs.AsReadOnly(); + + private readonly double _beta; + + /// + /// Localization root cause for multi-dimensional anomaly. + /// + /// The estimator's local . + /// Weight for generating score. + /// The name of the columns (first item of the tuple), and the name of the resulting output column (second item of the tuple). + + internal DTRootCauseLocalizationTransformer(IHostEnvironment env,double beta = DTRootCauseLocalizationEstimator.Defaults.Beta, params (string outputColumnName, string inputColumnName)[] columns) + : base(Contracts.CheckRef(env, nameof(env)).Register(RegistrationName), columns) + { + Host.CheckUserArg(beta >=0 && beta <= 1, nameof(Options.Beta), "Must be in [0,1]"); + + _beta = beta; + } + + // Factory method for SignatureDataTransform. + internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) + { + Contracts.CheckValue(env, nameof(env)); + env.CheckValue(options, nameof(options)); + env.CheckValue(input, nameof(input)); + env.CheckValue(options.Columns, nameof(options.Columns)); + + return new DTRootCauseLocalizationTransformer(env,options.Beta, options.Columns.Select(x => (x.Name, x.Source ?? x.Name)).ToArray()) + .MakeDataTransform(input); + } + + // Factory method for SignatureLoadModel. + private static DTRootCauseLocalizationTransformer Create(IHostEnvironment env, ModelLoadContext ctx) + { + Contracts.CheckValue(env, nameof(env)); + var host = env.Register(RegistrationName); + host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(GetVersionInfo()); + return new DTRootCauseLocalizationTransformer(host, ctx); + } + + private DTRootCauseLocalizationTransformer(IHost host, ModelLoadContext ctx) + : base(host, ctx) + { + } + + // Factory method for SignatureLoadDataTransform. + private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) + => Create(env, ctx).MakeDataTransform(input); + + // Factory method for SignatureLoadRowMapper. + private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema) + => Create(env, ctx).MakeRowMapper(inputSchema); + + private protected override void SaveModel(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); + + // *** Binary format *** + // + base.SaveColumns(ctx); + } + + private protected override IRowMapper MakeRowMapper(DataViewSchema schema) => new Mapper(this, schema); + + private protected override void CheckInputColumn(DataViewSchema inputSchema, int col, int srcCol) + { + if (!(inputSchema[srcCol].Type is RootCauseLocalizationInputDataViewType)) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", ColumnPairs[col].inputColumnName, "RootCauseLocalizationInputDataViewType", inputSchema[srcCol].Type.ToString()); + } + + private sealed class Mapper : OneToOneMapperBase + { + private DTRootCauseLocalizationTransformer _parent; + + public Mapper(DTRootCauseLocalizationTransformer parent, DataViewSchema inputSchema) + : base(parent.Host.Register(nameof(Mapper)), parent, inputSchema) + { + _parent = parent; + } + + protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore() + { + var result = new DataViewSchema.DetachedColumn[_parent.ColumnPairs.Length]; + for (int i = 0; i < _parent.ColumnPairs.Length; i++) + { + InputSchema.TryGetColumnIndex(_parent.ColumnPairs[i].inputColumnName, out int colIndex); + Host.Assert(colIndex >= 0); + + DataViewType type; + type = new RootCauseDataViewType(); + + result[i] = new DataViewSchema.DetachedColumn(_parent.ColumnPairs[i].outputColumnName, type, null); + } + return result; + } + + protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func activeOutput, out Action disposer) + { + Contracts.AssertValue(input); + Contracts.Assert(0 <= iinfo && iinfo < _parent.ColumnPairs.Length); + + var src = default(RootCauseLocalizationInput); + var getSrc = input.GetGetter(input.Schema[ColMapNewToOld[iinfo]]); + + disposer = + () => + { + if (src != null) + { + src.Dispose(); + src = null; + } + }; + + ValueGetter del = + (ref RootCause dst) => + { + getSrc(ref src); + if (src == null) + return; + + if (src.Slices.Count < 1) { + throw Host.Except($"Length of Slices must be larger than 0"); + } + //todo- more checks will be added here for the input + + dst = new RootCause(); + //dst.Items = new List{ new RootCauseItems() }; + //todo- algorithms would be implememted here + }; + + return del; + } + } + } + + /// + /// for the . + /// + /// + /// | + /// | Output column data type | | + /// | Exportable to ONNX | No | + /// + /// The resulting creates a new column, named as specified in the output column name parameters, and + /// localize the root causes which contribute most to the anomaly. + /// Check the See Also section for links to usage examples. + /// ]]> + /// + /// + /// + public sealed class DTRootCauseLocalizationEstimator : TrivialEstimator + { + internal static class Defaults + { + public const double Beta = 0.5; + } + + public enum AggregateType + { + /// + /// Make the aggregate type as sum. + /// + Sum = 0, + /// + /// Make the aggregate type as average. + /// + Avg = 1, + /// + /// Make the aggregate type as min. + /// + Min = 2, + /// + /// Make the aggregate type as max. + /// + Max = 3 + } + + /// + /// Localize root cause. + /// + /// The estimator's local . + /// The name of the columns (first item of the tuple), and the name of the resulting output column (second item of the tuple). + /// The weight for generating score in output result. + [BestFriend] + internal DTRootCauseLocalizationEstimator(IHostEnvironment env, double beta = Defaults.Beta,params(string outputColumnName, string inputColumnName)[] columns) + : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(DTRootCauseLocalizationEstimator)), new DTRootCauseLocalizationTransformer(env, beta,columns)) + { + } + + /// + /// Returns the of the schema which will be produced by the transformer. + /// Used for schema propagation and verification in a pipeline. + /// + public override SchemaShape GetOutputSchema(SchemaShape inputSchema) + { + Host.CheckValue(inputSchema, nameof(inputSchema)); + var result = inputSchema.ToDictionary(x => x.Name); + foreach (var colInfo in Transformer.Columns) + { + if (!inputSchema.TryFindColumn(colInfo.inputColumnName, out var col)) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colInfo.inputColumnName); + if (!(col.ItemType is RootCauseLocalizationInputDataViewType) || col.Kind != SchemaShape.Column.VectorKind.Scalar) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colInfo.inputColumnName, new RootCauseLocalizationInputDataViewType().ToString(), col.GetTypeString()); + + result[colInfo.outputColumnName] = new SchemaShape.Column(colInfo.outputColumnName, col.Kind, col.ItemType, col.IsKey, col.Annotations); + } + + return new SchemaShape(result.Values); + } + } +} diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 161f8bc27b..4739cec3ad 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -225,6 +225,23 @@ private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocaliza host.CheckUserArg(containsAnomalyTimestamp, nameof(src.Slices), "Has no points in the given anomaly timestamp"); } + /// + /// Create , which localizes root causess using decision tree algorithm. + /// + /// The transform's catalog. + /// Name of the column resulting from the transformation of . + /// Name of column to transform. + /// The weight parameter in score. The range of the parameter should be in [0,1]. + /// + /// + /// + /// + /// + public static DTRootCauseLocalizationEstimator LocalizeRootCauseByDT(this TransformsCatalog catalog, string outputColumnName, string inputColumnName = null, double beta=0.5) + => new DTRootCauseLocalizationEstimator(CatalogUtils.GetEnvironment(catalog),beta, new[] { (outputColumnName, inputColumnName ?? outputColumnName) }); + /// /// Singular Spectrum Analysis (SSA) model for univariate time-series forecasting. /// For the details of the model, refer to http://arxiv.org/pdf/1206.6910.pdf. diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index d25b614d37..f2c29c5a78 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Drawing; using System.IO; using Microsoft.ML.Data; using Microsoft.ML.TestFramework; From f9af0738cf5bde0b76f8f7a5afd945cd3fbeebd6 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:15:39 +0800 Subject: [PATCH 02/36] merge with remote repo --- .../TimeSeries/LocalizeRootCauseByDT.cs | 76 --- .../DTRootCauseLocalization.cs | 490 ------------------ .../TimeSeriesDirectApi.cs | 5 +- 3 files changed, 4 insertions(+), 567 deletions(-) delete mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseByDT.cs delete mode 100644 src/Microsoft.ML.TimeSeries/DTRootCauseLocalization.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseByDT.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseByDT.cs deleted file mode 100644 index 94d117fd45..0000000000 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseByDT.cs +++ /dev/null @@ -1,76 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Drawing; -using System.IO; -using Microsoft.ML; -using Microsoft.ML.Data; -using Microsoft.ML.Transforms.TimeSeries; - -namespace Samples.Dynamic -{ - public static class LocalizeRootCause - { - public static void Example() - { - // Create a new ML context, for ML.NET operations. It can be used for - // exception tracking and logging, as well as the source of randomness. - var mlContext = new MLContext(); - - // Create an empty list as the dataset. The 'NormalizeText' API does not - // require training data as the estimator ('TextNormalizingEstimator') - // created by 'NormalizeText' API is not a trainable estimator. The - // empty list is only needed to pass input schema to the pipeline. - var emptySamples = new List(); - - // Convert sample list to an empty IDataView. - var emptyDataView = mlContext.Data.LoadFromEnumerable(emptySamples); - - // A pipeline for localizeing root cause. - var localizePipeline = mlContext.Transforms.LocalizeRootCauseByDT(nameof(RootCauseLocalizationTransformedData.RootCause), nameof(RootCauseLocalizationData.Input)); - - // Fit to data. - var localizeTransformer = localizePipeline.Fit(emptyDataView); - - // Create the prediction engine to get the root cause result from the - // input data. - var predictionEngine = mlContext.Model.CreatePredictionEngine(localizeTransformer); - - // Call the prediction API. - var data = new RootCauseLocalizationData(new DateTime(), new Dictionary(), new List() { new MetricSlice(new DateTime(), new List()) }, "SUM", "SUM"); - - var prediction = predictionEngine.Predict(data); - - // Print the localization result. - Console.WriteLine($"Localized result: {prediction.RootCause}"); - } - - - private class RootCauseLocalizationData - { - [RootCauseLocalizationInputType] - public RootCauseLocalizationInput Input { get; set; } - - public RootCauseLocalizationData() - { - Input = null; - } - - public RootCauseLocalizationData(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices,String aggregateType, string aggregateSymbol) - { - Input = new RootCauseLocalizationInput(anomalyTimestamp, anomalyDimensions, slices, DTRootCauseLocalizationEstimator.AggregateType.Sum, aggregateSymbol); - } - } - - private class RootCauseLocalizationTransformedData - { - [RootCauseType()] - public RootCause RootCause { get; set; } - - public RootCauseLocalizationTransformedData() - { - RootCause = null; - } - } - } -} diff --git a/src/Microsoft.ML.TimeSeries/DTRootCauseLocalization.cs b/src/Microsoft.ML.TimeSeries/DTRootCauseLocalization.cs deleted file mode 100644 index c70dd398ff..0000000000 --- a/src/Microsoft.ML.TimeSeries/DTRootCauseLocalization.cs +++ /dev/null @@ -1,490 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Microsoft.ML; -using Microsoft.ML.CommandLine; -using Microsoft.ML.Data; -using Microsoft.ML.Internal.Utilities; -using Microsoft.ML.Runtime; -using Microsoft.ML.Transforms.TimeSeries; - -[assembly: LoadableClass(DTRootCauseLocalizationTransformer.Summary, typeof(IDataTransform), typeof(DTRootCauseLocalizationTransformer), typeof(DTRootCauseLocalizationTransformer.Options), typeof(SignatureDataTransform), - DTRootCauseLocalizationTransformer.UserName, "DTRootCauseLocalizationTransform", "DTRootCauseLocalization")] - -[assembly: LoadableClass(DTRootCauseLocalizationTransformer.Summary, typeof(IDataTransform), typeof(DTRootCauseLocalizationTransformer), null, typeof(SignatureLoadDataTransform), - DTRootCauseLocalizationTransformer.UserName, DTRootCauseLocalizationTransformer.LoaderSignature)] - -[assembly: LoadableClass(typeof(DTRootCauseLocalizationTransformer), null, typeof(SignatureLoadModel), - DTRootCauseLocalizationTransformer.UserName, DTRootCauseLocalizationTransformer.LoaderSignature)] - -[assembly: LoadableClass(typeof(IRowMapper), typeof(DTRootCauseLocalizationTransformer), null, typeof(SignatureLoadRowMapper), - DTRootCauseLocalizationTransformer.UserName, DTRootCauseLocalizationTransformer.LoaderSignature)] - -namespace Microsoft.ML.Transforms.TimeSeries -{ - public sealed class RootCauseLocalizationInputTypeAttribute : DataViewTypeAttribute - { - /// - /// Create a root cause localizagin input type. - /// - public RootCauseLocalizationInputTypeAttribute() - { - } - - /// - /// Equal function. - /// - public override bool Equals(DataViewTypeAttribute other) - { - if (!(other is RootCauseLocalizationInputTypeAttribute otherAttribute)) - return false; - return true; - } - - /// - /// Produce the same hash code for all RootCauseLocalizationInputTypeAttribute. - /// - public override int GetHashCode() - { - return 0; - } - - public override void Register() - { - DataViewTypeManager.Register(new RootCauseLocalizationInputDataViewType(), typeof(RootCauseLocalizationInput), this); - } - } - - public sealed class RootCauseTypeAttribute : DataViewTypeAttribute - { - /// - /// Create an root cause type. - /// - public RootCauseTypeAttribute() - { - } - - /// - /// RootCauseTypeAttribute with the same type should equal. - /// - public override bool Equals(DataViewTypeAttribute other) - { - if (other is RootCauseTypeAttribute otherAttribute) - return true; - return false; - } - - /// - /// Produce the same hash code for all RootCauseTypeAttribute. - /// - public override int GetHashCode() - { - return 0; - } - - public override void Register() - { - DataViewTypeManager.Register(new RootCauseDataViewType(), typeof(RootCause), this); - } - } - - public sealed class RootCause - { - public List Items { get; set; } - } - - public sealed class RootCauseItems { - public double Score; - public List Path; - public Dictionary RootCause; - public AnomalyDirection Direction; - } - - public enum AnomalyDirection { - /// - /// the value is larger than expected value. - /// - Up = 0, - /// - /// the value is lower than expected value. - /// - Down = 1 - } - - public sealed class RootCauseLocalizationInput - { - public DateTime AnomalyTimestamp { get; set; } - - public Dictionary AnomalyDimensions { get; set; } - - public List Slices { get; set; } - - public DTRootCauseLocalizationEstimator.AggregateType AggType{ get; set; } - - public string AggSymbol { get; set; } - - public RootCauseLocalizationInput(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, DTRootCauseLocalizationEstimator.AggregateType aggregateType, string aggregateSymbol) { - AnomalyTimestamp = anomalyTimestamp; - AnomalyDimensions = anomalyDimensions; - Slices = slices; - AggType = aggregateType; - AggSymbol = aggregateSymbol; - } - public void Dispose() - { - AnomalyDimensions = null; - Slices = null; - } - } - - public sealed class MetricSlice - { - public DateTime TimeStamp { get; set; } - public List Points { get; set; } - - public MetricSlice(DateTime timeStamp, List points) { - TimeStamp = timeStamp; - Points = points; - } - } - - public sealed class Point { - public double Value { get; set; } - public double ExpectedValue { get; set; } - public bool IsAnomaly { get; set; } - public Dictionary Dimensions{ get; set; } - } - - public sealed class RootCauseDataViewType : StructuredDataViewType - { - public RootCauseDataViewType() - : base(typeof(RootCause)) - { - } - - public override bool Equals(DataViewType other) - { - if (other == this) - return true; - if (!(other is RootCauseDataViewType tmp)) - return false; - return true; - } - - public override int GetHashCode() - { - return 0; - } - - public override string ToString() - { - return typeof(RootCauseDataViewType).Name; - } - } - - public sealed class RootCauseLocalizationInputDataViewType : StructuredDataViewType - { - public RootCauseLocalizationInputDataViewType() - : base(typeof(RootCauseLocalizationInput)) - { - } - - public override bool Equals(DataViewType other) - { - if (!(other is RootCauseLocalizationInputDataViewType tmp)) - return false; - return true; - } - - public override int GetHashCode() - { - return 0; - } - - public override string ToString() - { - return typeof(RootCauseLocalizationInputDataViewType).Name; - } - } - - // REVIEW: Rewrite as LambdaTransform to simplify. - // REVIEW: Should it be separate transform or part of ImageResizerTransform? - /// - /// resulting from fitting an . - /// - public sealed class DTRootCauseLocalizationTransformer : OneToOneTransformerBase - { - internal sealed class Column : OneToOneColumn - { - internal static Column Parse(string str) - { - var res = new Column(); - if (res.TryParse(str)) - return res; - return null; - } - - internal bool TryUnparse(StringBuilder sb) - { - Contracts.AssertValue(sb); - return TryUnparseCore(sb); - } - } - - internal class Options : TransformInputBase - { - [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", Name = "Column", ShortName = "col", SortOrder = 1)] - public Column[] Columns; - - [Argument(ArgumentType.AtMostOnce, HelpText = "Weight for getting the score for the root cause item.", ShortName = "Beta", SortOrder = 2)] - public double Beta = DTRootCauseLocalizationEstimator.Defaults.Beta; - - } - - internal const string Summary = "Localize root cause for anomaly."; - - internal const string UserName = "DT Root Cause Localization Transform"; - internal const string LoaderSignature = "DTRootCauseLTransform"; - - private static VersionInfo GetVersionInfo() - { - return new VersionInfo( - modelSignature: "DTRCL", - verWrittenCur: 0x00010001, // Initial - verReadableCur: 0x00010001, - verWeCanReadBack: 0x00010001, - loaderSignature: LoaderSignature, - loaderAssemblyName: typeof(DTRootCauseLocalizationTransformer).Assembly.FullName); - } - - private const string RegistrationName = "RootCauseLocalization"; - - /// - /// The input and output column pairs passed to this . - /// - internal IReadOnlyCollection<(string outputColumnName, string inputColumnName)> Columns => ColumnPairs.AsReadOnly(); - - private readonly double _beta; - - /// - /// Localization root cause for multi-dimensional anomaly. - /// - /// The estimator's local . - /// Weight for generating score. - /// The name of the columns (first item of the tuple), and the name of the resulting output column (second item of the tuple). - - internal DTRootCauseLocalizationTransformer(IHostEnvironment env,double beta = DTRootCauseLocalizationEstimator.Defaults.Beta, params (string outputColumnName, string inputColumnName)[] columns) - : base(Contracts.CheckRef(env, nameof(env)).Register(RegistrationName), columns) - { - Host.CheckUserArg(beta >=0 && beta <= 1, nameof(Options.Beta), "Must be in [0,1]"); - - _beta = beta; - } - - // Factory method for SignatureDataTransform. - internal static IDataTransform Create(IHostEnvironment env, Options options, IDataView input) - { - Contracts.CheckValue(env, nameof(env)); - env.CheckValue(options, nameof(options)); - env.CheckValue(input, nameof(input)); - env.CheckValue(options.Columns, nameof(options.Columns)); - - return new DTRootCauseLocalizationTransformer(env,options.Beta, options.Columns.Select(x => (x.Name, x.Source ?? x.Name)).ToArray()) - .MakeDataTransform(input); - } - - // Factory method for SignatureLoadModel. - private static DTRootCauseLocalizationTransformer Create(IHostEnvironment env, ModelLoadContext ctx) - { - Contracts.CheckValue(env, nameof(env)); - var host = env.Register(RegistrationName); - host.CheckValue(ctx, nameof(ctx)); - ctx.CheckAtModel(GetVersionInfo()); - return new DTRootCauseLocalizationTransformer(host, ctx); - } - - private DTRootCauseLocalizationTransformer(IHost host, ModelLoadContext ctx) - : base(host, ctx) - { - } - - // Factory method for SignatureLoadDataTransform. - private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) - => Create(env, ctx).MakeDataTransform(input); - - // Factory method for SignatureLoadRowMapper. - private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema) - => Create(env, ctx).MakeRowMapper(inputSchema); - - private protected override void SaveModel(ModelSaveContext ctx) - { - Host.CheckValue(ctx, nameof(ctx)); - - ctx.CheckAtModel(); - ctx.SetVersionInfo(GetVersionInfo()); - - // *** Binary format *** - // - base.SaveColumns(ctx); - } - - private protected override IRowMapper MakeRowMapper(DataViewSchema schema) => new Mapper(this, schema); - - private protected override void CheckInputColumn(DataViewSchema inputSchema, int col, int srcCol) - { - if (!(inputSchema[srcCol].Type is RootCauseLocalizationInputDataViewType)) - throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", ColumnPairs[col].inputColumnName, "RootCauseLocalizationInputDataViewType", inputSchema[srcCol].Type.ToString()); - } - - private sealed class Mapper : OneToOneMapperBase - { - private DTRootCauseLocalizationTransformer _parent; - - public Mapper(DTRootCauseLocalizationTransformer parent, DataViewSchema inputSchema) - : base(parent.Host.Register(nameof(Mapper)), parent, inputSchema) - { - _parent = parent; - } - - protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore() - { - var result = new DataViewSchema.DetachedColumn[_parent.ColumnPairs.Length]; - for (int i = 0; i < _parent.ColumnPairs.Length; i++) - { - InputSchema.TryGetColumnIndex(_parent.ColumnPairs[i].inputColumnName, out int colIndex); - Host.Assert(colIndex >= 0); - - DataViewType type; - type = new RootCauseDataViewType(); - - result[i] = new DataViewSchema.DetachedColumn(_parent.ColumnPairs[i].outputColumnName, type, null); - } - return result; - } - - protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func activeOutput, out Action disposer) - { - Contracts.AssertValue(input); - Contracts.Assert(0 <= iinfo && iinfo < _parent.ColumnPairs.Length); - - var src = default(RootCauseLocalizationInput); - var getSrc = input.GetGetter(input.Schema[ColMapNewToOld[iinfo]]); - - disposer = - () => - { - if (src != null) - { - src.Dispose(); - src = null; - } - }; - - ValueGetter del = - (ref RootCause dst) => - { - getSrc(ref src); - if (src == null) - return; - - if (src.Slices.Count < 1) { - throw Host.Except($"Length of Slices must be larger than 0"); - } - //todo- more checks will be added here for the input - - dst = new RootCause(); - //dst.Items = new List{ new RootCauseItems() }; - //todo- algorithms would be implememted here - }; - - return del; - } - } - } - - /// - /// for the . - /// - /// - /// | - /// | Output column data type | | - /// | Exportable to ONNX | No | - /// - /// The resulting creates a new column, named as specified in the output column name parameters, and - /// localize the root causes which contribute most to the anomaly. - /// Check the See Also section for links to usage examples. - /// ]]> - /// - /// - /// - public sealed class DTRootCauseLocalizationEstimator : TrivialEstimator - { - internal static class Defaults - { - public const double Beta = 0.5; - } - - public enum AggregateType - { - /// - /// Make the aggregate type as sum. - /// - Sum = 0, - /// - /// Make the aggregate type as average. - /// - Avg = 1, - /// - /// Make the aggregate type as min. - /// - Min = 2, - /// - /// Make the aggregate type as max. - /// - Max = 3 - } - - /// - /// Localize root cause. - /// - /// The estimator's local . - /// The name of the columns (first item of the tuple), and the name of the resulting output column (second item of the tuple). - /// The weight for generating score in output result. - [BestFriend] - internal DTRootCauseLocalizationEstimator(IHostEnvironment env, double beta = Defaults.Beta,params(string outputColumnName, string inputColumnName)[] columns) - : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(DTRootCauseLocalizationEstimator)), new DTRootCauseLocalizationTransformer(env, beta,columns)) - { - } - - /// - /// Returns the of the schema which will be produced by the transformer. - /// Used for schema propagation and verification in a pipeline. - /// - public override SchemaShape GetOutputSchema(SchemaShape inputSchema) - { - Host.CheckValue(inputSchema, nameof(inputSchema)); - var result = inputSchema.ToDictionary(x => x.Name); - foreach (var colInfo in Transformer.Columns) - { - if (!inputSchema.TryFindColumn(colInfo.inputColumnName, out var col)) - throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colInfo.inputColumnName); - if (!(col.ItemType is RootCauseLocalizationInputDataViewType) || col.Kind != SchemaShape.Column.VectorKind.Scalar) - throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colInfo.inputColumnName, new RootCauseLocalizationInputDataViewType().ToString(), col.GetTypeString()); - - result[colInfo.outputColumnName] = new SchemaShape.Column(colInfo.outputColumnName, col.Kind, col.ItemType, col.IsKey, col.Annotations); - } - - return new SchemaShape(result.Values); - } - } -} diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index f2c29c5a78..32d88ce5cf 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -4,7 +4,7 @@ using System; using System.Collections.Generic; -using System.Drawing; +using System.Data; using System.IO; using Microsoft.ML.Data; using Microsoft.ML.TestFramework; @@ -12,6 +12,9 @@ using Microsoft.ML.Transforms.TimeSeries; using Xunit; using Xunit.Abstractions; +using Microsoft.VisualBasic.CompilerServices; + +using Microsoft.VisualBasic.FileIO; namespace Microsoft.ML.Tests { From 0551ecdcf7cef6ee9c5888bab5a7b2ea905bd946 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:25:40 +0800 Subject: [PATCH 03/36] rebase --- src/Native/MatrixFactorizationNative/libmf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Native/MatrixFactorizationNative/libmf b/src/Native/MatrixFactorizationNative/libmf index 403153ca20..298715a4e4 160000 --- a/src/Native/MatrixFactorizationNative/libmf +++ b/src/Native/MatrixFactorizationNative/libmf @@ -1 +1 @@ -Subproject commit 403153ca204817e2901b2872d977088316360641 +Subproject commit 298715a4e458bc09c6a27c8643a58095afbdadf1 From 3138c39f569d4b99f021a003071cb2893e2d94c9 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:37:43 +0800 Subject: [PATCH 04/36] merge --- .../TimeSeries/LocalizeRootCauseEvaluation.cs | 403 ++++++++++++++++++ .../Microsoft.ML.Samples.csproj | 3 +- docs/samples/Microsoft.ML.Samples/Program.cs | 16 +- .../TimeSeriesDirectApi.cs | 26 -- 4 files changed, 415 insertions(+), 33 deletions(-) create mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs new file mode 100644 index 0000000000..e2e0e5c985 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs @@ -0,0 +1,403 @@ +using System; +using System.Collections.Generic; +using System.Data; +using Microsoft.ML; +using Microsoft.ML.TimeSeries; +using Microsoft.ML.Transforms.TimeSeries; + +using Microsoft.VisualBasic.FileIO; + +namespace Samples.Dynamic.Transforms.TimeSeries +{ + public static class LocalizeRootCauseEvaluation + { + public static void Example() + { + Dictionary> rootNodeMap = GetAnomalyRootMap(); + Dictionary>> labeledRootCauseMap = GetLabeledRootCauseMap(); + + string aggSymbol = "##EMPTY##awqegp##"; + + int totalTp = 0; + int totalFp = 0; + int totalFn = 0; + int totalCount = 0; + + bool exactly = false; + + foreach (KeyValuePair> item in rootNodeMap) + { + DateTime timeStamp = item.Key; + + int seconds = Convert.ToInt32(timeStamp.Subtract(new DateTime(1970, 1, 1, 0, 0, 0, 0)).TotalSeconds); + string path = String.Format("D:/rootcause/Dataset_yaniv/raw_data_201908_202002/{0}.csv", seconds); + List points = GetPoints(path); + List slices = new List(); + slices.Add(new MetricSlice(timeStamp, points)); + + PredictionEngine engine = GetRootCausePredictionEngine(); + + var newRootCauseInput = new RootCauseLocalizationData(timeStamp, rootNodeMap[timeStamp], new List() { new MetricSlice(timeStamp, points) }, DTRootCauseLocalizationEstimator.AggregateType.Sum, aggSymbol); + + List list = new List(); + GetRootCause(list, newRootCauseInput, engine); + + List> labeledRootCause = labeledRootCauseMap[timeStamp]; + List> detectedRootCause = ConvertRootCauseItemToDic(list); + RemoveAggSymbol(detectedRootCause, aggSymbol); + + Tuple evaluation = ScoreRootCause(detectedRootCause, labeledRootCause, exactly, timeStamp); + totalTp += evaluation.Item1; + totalFp += evaluation.Item2; + totalFn += evaluation.Item3; + totalCount++; + } + + double precision = (double)totalTp / (totalTp + totalFp); + double recall = (double)totalTp / (totalTp + totalFn); + double f1 = 2 * precision * recall / (precision + recall); + Console.WriteLine(String.Format("Total Count : {0}, TP: {1}, FP: {2}, FN: {3}", totalCount, totalTp, totalFp, totalFn)); + Console.WriteLine(String.Format("Precision : {0}, Recall: {1}, F1: {2}", precision, recall, f1)); + } + + private static Tuple ScoreRootCause(List> detectedRootCause, List> labeledRootCause, bool exactly, DateTime timeStamp) + { + int tp = 0; + int fp = 0; + int fn; ; + List labelSet = new List(); + foreach (Dictionary cause in detectedRootCause) + { + string tpCause = FindTruePositive(cause, labeledRootCause, exactly); + if (tpCause == null) + { + //todo - seriesalize the root cause + Console.WriteLine(String.Format("FP : timestamp - {0}, detected root cause ", timeStamp)); + Console.WriteLine(string.Join(Environment.NewLine, cause)); + } + else + { + tp++; + labelSet.Add(tpCause); + } + } + + fn = labeledRootCause.Count - labelSet.Count; + if (fn != 0) + { + List> nCause = GetNegtiveCause(labeledRootCause, labelSet); + //todo seralize + if (nCause.Count > 0) + { + Console.WriteLine(String.Format("FN : timestamp - {0}", timeStamp)); + foreach (Dictionary cause in nCause) + { + Console.WriteLine(string.Join(Environment.NewLine, nCause)); + } + } + } + + return new Tuple(tp, fp, fn); + } + + private static List> GetNegtiveCause(List> labelCauses, List labelSet) + { + List> causeList = new List>(); + foreach (Dictionary cause in labelCauses) + { + if (!labelSet.Contains(GetDicHashCode(cause))) + { + causeList.Add(cause); + } + } + return causeList; + } + + private static string FindTruePositive(Dictionary cause, List> labelCauses, bool exactly) + { + foreach (Dictionary label in labelCauses) + { + string id = GetDicHashCode(label); + int compare = CompareCause(cause, label); + if (compare == 0) + { + return id; + } + else if (!exactly && (compare == 1 || compare == 2)) + { + return id; + } + } + return null; + } + + + private static string GetDicHashCode(Dictionary dic) + { + return dic.GetHashCode().ToString(); + } + + private static int CompareCause(Dictionary detect, Dictionary label) + { + + if (detect.Equals(label)) + { + return 0; + } + else if (DTRootCauseLocalizationUtils.ContainsAll(detect, label)) + { + return 1; + } + else if (DTRootCauseLocalizationUtils.ContainsAll(label, detect)) + { + return 2; + } + return 3; + } + private static List> ConvertRootCauseItemToDic(List items) + { + List> list = new List>(); + foreach (RootCauseItem item in items) + { + list.Add(item.RootCause); + } + return list; + } + + private static void RemoveAggSymbol(List> dimensions, string aggSymbol) + { + foreach (Dictionary dim in dimensions) + { + foreach (string key in dim.Keys) + { + if (dim[key].Equals(aggSymbol)) + { + dim.Remove(key); + } + } + } + } + + private static PredictionEngine GetRootCausePredictionEngine() + { + //// Create an root cause localizatiom input list from csv. + var rootCauseLocalizationData = new List() { new RootCauseLocalizationData(new DateTime(), new Dictionary(), new List() { new MetricSlice(new DateTime(), new List()) }, DTRootCauseLocalizationEstimator.AggregateType.Sum, "SUM") }; + + + var ml = new MLContext(1); + // Convert the list of root cause data to an IDataView object, which is consumable by ML.NET API. + var data = ml.Data.LoadFromEnumerable(rootCauseLocalizationData); + + // Create pipeline to localize root cause by decision tree. + var pipeline = ml.Transforms.LocalizeRootCauseByDT(nameof(RootCauseLocalizationTransformedData.RootCause), nameof(RootCauseLocalizationData.Input)); + + // Fit the model. + var model = pipeline.Fit(data); + + // Test path: input list -> IDataView -> Enumerable of RootCauseLocalizationInputs. + var transformedData = model.Transform(data); + + // Load input list in DataView back to Enumerable. + var transformedDataPoints = ml.Data.CreateEnumerable(transformedData, false); + + foreach (var dataPoint in transformedDataPoints) + { + var rootCause = dataPoint.RootCause; + } + + var engine = ml.Model.CreatePredictionEngine(model); + return engine; + } + + private static string _ocsDataCenter = "OCSDatacenter"; + private static string _appType = "AppType"; + private static string _releaseAudienceGroup = "Release_AudienceGroup"; + private static string _wacDatacenter = "WACDatacenter"; + private static string _requestType = "RequestType"; + private static string _statusCode = "StatusCode"; + + private static List _dimensionKeys = new List() { _ocsDataCenter, _appType, _releaseAudienceGroup, _wacDatacenter, _statusCode, _requestType }; + + private static Dictionary> GetAnomalyRootMap() + { + var anomalyRootData = GetDataTabletFromCSVFile("D:/rootcause/Dataset_yaniv/root_cause_201908_202002/anomaly_root.csv"); + + Dictionary> rootNodeMap = new Dictionary>(); + foreach (DataRow row in anomalyRootData.Rows) + { + // load the data, build the RootCauseInput, take care of empty value + long seconds = long.Parse(row["TimeStamp"].ToString()); + DateTime t = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); + + Dictionary dimension = new Dictionary(); + foreach (string key in _dimensionKeys) + { + if (!row[key].ToString().Equals("")) + { + dimension.Add(key, row[key].ToString()); + } + } + + rootNodeMap.Add(t, dimension); + } + return rootNodeMap; + } + + private static Dictionary>> GetLabeledRootCauseMap() + { + var labeldRootCause = GetDataTabletFromCSVFile("D:/rootcause/Dataset_yaniv/root_cause_201908_202002/labeled_root_cause.csv"); + + Dictionary>> map = new Dictionary>>(); + foreach (DataRow row in labeldRootCause.Rows) + { + // load the data, build the labled result, take care of empty value + long seconds = long.Parse(row["TimeStamp"].ToString()); + DateTime t = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); + + Dictionary dimension = new Dictionary(); + foreach (string key in _dimensionKeys) + { + if (!row[key].ToString().Equals("")) + { + dimension.Add(key, row[key].ToString()); + } + } + + if (map.ContainsKey(t)) + { + map[t].Add(dimension); + } + else + { + map.Add(t, new List>() { dimension }); + } + } + return map; + } + + private static List GetPoints(string path) + { + + + var inputData = GetDataTabletFromCSVFile(path); + + DateTime timeStamp = new DateTime(); + + List points = new List(); + foreach (DataRow row in inputData.Rows) + { + // load the data, build the RootCauseInput, take care of empty value + long seconds = long.Parse(row["TimeStamp"].ToString()); + timeStamp = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); + double value = Double.Parse(row["Value"].ToString()); + double expectedValue = 0; + if (!row["ExpectedValue"].ToString().Equals("")) + { + expectedValue = Double.Parse(row["ExpectedValue"].ToString()); + } + bool isAnomaly = Boolean.Parse(row["IsAnomaly"].ToString()); + Dictionary dimension = new Dictionary(); + foreach (string key in _dimensionKeys) + { + if (!row[key].ToString().Equals("")) + { + dimension.Add(key, row[key].ToString()); + } + } + + points.Add(new Microsoft.ML.TimeSeries.Point(value, expectedValue, isAnomaly, dimension)); ; + } + + return points; + } + + private static void GetRootCause(List rootCauseList, RootCauseLocalizationData inputData, PredictionEngine engine) + { + + RootCauseLocalizationTransformedData incrementalResult = engine.Predict(inputData); + + if (incrementalResult.RootCause.Items.Count == 0 || ( + incrementalResult.RootCause.Items.Count == 1 && incrementalResult.RootCause.Items[0].RootCause.Equals(inputData.Input.AnomalyDimensions) + )) + { + if (!rootCauseList.Contains(new RootCauseItem(inputData.Input.AnomalyDimensions))) + { + rootCauseList.Add(new RootCauseItem(inputData.Input.AnomalyDimensions)); + + } + return; + } + else + { + foreach (RootCauseItem item in incrementalResult.RootCause.Items) + { + RootCauseLocalizationData newData = new RootCauseLocalizationData(inputData.Input.AnomalyTimestamp, + item.RootCause, inputData.Input.Slices, inputData.Input.AggType, inputData.Input.AggSymbol); + GetRootCause(rootCauseList, newData, engine); + } + } + } + + private static DataTable GetDataTabletFromCSVFile(string filePath) + { + DataTable csvData = new DataTable(); + + + using (TextFieldParser csvReader = new TextFieldParser(filePath)) + { + csvReader.SetDelimiters(new string[] { "," }); + csvReader.HasFieldsEnclosedInQuotes = true; + string[] colFields = csvReader.ReadFields(); + foreach (string column in colFields) + { + DataColumn datecolumn = new DataColumn(column); + datecolumn.AllowDBNull = true; + csvData.Columns.Add(datecolumn); + } + + while (!csvReader.EndOfData) + { + string[] fieldData = csvReader.ReadFields(); + //Making empty value as null + for (int i = 0; i < fieldData.Length; i++) + { + if (fieldData[i] == "") + { + fieldData[i] = null; + } + } + csvData.Rows.Add(fieldData); + } + } + + return csvData; + } + + private class RootCauseLocalizationData + { + [RootCauseLocalizationInputType] + public RootCauseLocalizationInput Input { get; set; } + + public RootCauseLocalizationData() + { + Input = null; + } + + public RootCauseLocalizationData(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, DTRootCauseLocalizationEstimator.AggregateType aggregateteType, string aggregateSymbol) + { + Input = new RootCauseLocalizationInput(anomalyTimestamp, anomalyDimensions, slices, aggregateteType, aggregateSymbol); + } + } + + private class RootCauseLocalizationTransformedData + { + [RootCauseType()] + public RootCause RootCause { get; set; } + + public RootCauseLocalizationTransformedData() + { + RootCause = null; + } + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj index 5951c4bbd1..42f11558ea 100644 --- a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj +++ b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj @@ -1,7 +1,8 @@  - netcoreapp2.1 + + netcoreapp3.0 Exe false diff --git a/docs/samples/Microsoft.ML.Samples/Program.cs b/docs/samples/Microsoft.ML.Samples/Program.cs index 4c46399421..a3da984b16 100644 --- a/docs/samples/Microsoft.ML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.Samples/Program.cs @@ -13,16 +13,20 @@ internal static void RunAll() int samples = 0; foreach (var type in Assembly.GetExecutingAssembly().GetTypes()) { - var sample = type.GetMethod("Example", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); - - if (sample != null) + if (type.Name.Equals("LocalizeRootCauseEvaluation")) { - Console.WriteLine(type.Name); - sample.Invoke(null, null); - samples++; + var sample = type.GetMethod("Example", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); + + if (sample != null) + { + Console.WriteLine(type.Name); + sample.Invoke(null, null); + samples++; + } } } + Console.WriteLine("Number of samples that ran without any exception: " + samples); } } diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index 32d88ce5cf..fd853e39d5 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; -using System.Data; using System.IO; using Microsoft.ML.Data; using Microsoft.ML.TestFramework; @@ -12,9 +11,6 @@ using Microsoft.ML.Transforms.TimeSeries; using Xunit; using Xunit.Abstractions; -using Microsoft.VisualBasic.CompilerServices; - -using Microsoft.VisualBasic.FileIO; namespace Microsoft.ML.Tests { @@ -664,8 +660,6 @@ public void TestSrCnnBatchAnomalyDetector( k += 1; } } - - [Fact] public void RootCauseLocalization() { // Create an root cause localizatiom input @@ -673,99 +667,79 @@ public void RootCauseLocalization() var ml = new MLContext(1); RootCause rootCause = ml.AnomalyDetection.LocalizeRootCause(rootCauseLocalizationInput); - Assert.NotNull(rootCause); Assert.Equal(1, (int)rootCause.Items.Count); Assert.Equal(3, (int)rootCause.Items[0].Dimension.Count); Assert.Equal(AnomalyDirection.Up, rootCause.Items[0].Direction); Assert.Equal(1, (int)rootCause.Items[0].Path.Count); Assert.Equal("DataCenter", rootCause.Items[0].Path[0]); - Dictionary expectedDim = new Dictionary(); expectedDim.Add("Country", "UK"); expectedDim.Add("DeviceType", _rootCauseAggSymbol); expectedDim.Add("DataCenter", "DC1"); - foreach (KeyValuePair pair in rootCause.Items[0].Dimension) { Assert.Equal(expectedDim[pair.Key], pair.Value); } } - private static List GetRootCauseLocalizationPoints() { List points = new List(); - Dictionary dic1 = new Dictionary(); dic1.Add("Country", "UK"); dic1.Add("DeviceType", "Laptop"); dic1.Add("DataCenter", "DC1"); points.Add(new TimeSeriesPoint(200, 100, true, dic1)); - Dictionary dic2 = new Dictionary(); dic2.Add("Country", "UK"); dic2.Add("DeviceType", "Mobile"); dic2.Add("DataCenter", "DC1"); points.Add(new TimeSeriesPoint(1000, 100, true, dic2)); - Dictionary dic3 = new Dictionary(); dic3.Add("Country", "UK"); dic3.Add("DeviceType", _rootCauseAggSymbol); dic3.Add("DataCenter", "DC1"); points.Add(new TimeSeriesPoint(1200, 200, true, dic3)); - Dictionary dic4 = new Dictionary(); dic4.Add("Country", "UK"); dic4.Add("DeviceType", "Laptop"); dic4.Add("DataCenter", "DC2"); points.Add(new TimeSeriesPoint(100, 100, false, dic4)); - Dictionary dic5 = new Dictionary(); dic5.Add("Country", "UK"); dic5.Add("DeviceType", "Mobile"); dic5.Add("DataCenter", "DC2"); points.Add(new TimeSeriesPoint(200, 200, false, dic5)); - Dictionary dic6 = new Dictionary(); dic6.Add("Country", "UK"); dic6.Add("DeviceType", _rootCauseAggSymbol); dic6.Add("DataCenter", "DC2"); points.Add(new TimeSeriesPoint(300, 300, false, dic6)); - Dictionary dic7 = new Dictionary(); dic7.Add("Country", "UK"); dic7.Add("DeviceType", _rootCauseAggSymbol); dic7.Add("DataCenter", _rootCauseAggSymbol); points.Add(new TimeSeriesPoint(1500, 500, true, dic7)); - Dictionary dic8 = new Dictionary(); dic8.Add("Country", "UK"); dic8.Add("DeviceType", "Laptop"); dic8.Add("DataCenter", _rootCauseAggSymbol); points.Add(new TimeSeriesPoint(300, 200, true, dic8)); - Dictionary dic9 = new Dictionary(); dic9.Add("Country", "UK"); dic9.Add("DeviceType", "Mobile"); dic9.Add("DataCenter", _rootCauseAggSymbol); points.Add(new TimeSeriesPoint(1200, 300, true, dic9)); - return points; - } - private static Dictionary GetRootCauseAnomalyDimension() - { Dictionary dim = new Dictionary(); dim.Add("Country", "UK"); dim.Add("DeviceType", _rootCauseAggSymbol); dim.Add("DataCenter", _rootCauseAggSymbol); - return dim; } - private static DateTime GetRootCauseTimestamp() { return new DateTime(2020, 3, 23, 0, 0, 0); - } } } From 946ac43f8fad853bce22aea4e4a3d1ef6a2a9cc4 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Fri, 20 Mar 2020 11:08:15 +0800 Subject: [PATCH 05/36] temp save for internal review --- .../TimeSeries/LocalizeRootCauseEvaluation.cs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs index e2e0e5c985..f4c8f9f7be 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs @@ -71,9 +71,10 @@ private static Tuple ScoreRootCause(List ScoreRootCause(List> nCause = GetNegtiveCause(labeledRootCause, labelSet); - //todo seralize if (nCause.Count > 0) { Console.WriteLine(String.Format("FN : timestamp - {0}", timeStamp)); foreach (Dictionary cause in nCause) { - Console.WriteLine(string.Join(Environment.NewLine, nCause)); + Console.WriteLine(string.Join(Environment.NewLine, cause)); + Console.WriteLine("---------------------"); } + } } @@ -200,11 +202,6 @@ private static PredictionEngine(transformedData, false); - foreach (var dataPoint in transformedDataPoints) - { - var rootCause = dataPoint.RootCause; - } - var engine = ml.Model.CreatePredictionEngine(model); return engine; } From d39e65733ab330ce7e227ab82befe61810973389 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:40:12 +0800 Subject: [PATCH 06/36] merge --- .../TimeSeries/LocalizeRootCauseEvaluation.cs | 400 ------------------ 1 file changed, 400 deletions(-) delete mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs deleted file mode 100644 index f4c8f9f7be..0000000000 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs +++ /dev/null @@ -1,400 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Data; -using Microsoft.ML; -using Microsoft.ML.TimeSeries; -using Microsoft.ML.Transforms.TimeSeries; - -using Microsoft.VisualBasic.FileIO; - -namespace Samples.Dynamic.Transforms.TimeSeries -{ - public static class LocalizeRootCauseEvaluation - { - public static void Example() - { - Dictionary> rootNodeMap = GetAnomalyRootMap(); - Dictionary>> labeledRootCauseMap = GetLabeledRootCauseMap(); - - string aggSymbol = "##EMPTY##awqegp##"; - - int totalTp = 0; - int totalFp = 0; - int totalFn = 0; - int totalCount = 0; - - bool exactly = false; - - foreach (KeyValuePair> item in rootNodeMap) - { - DateTime timeStamp = item.Key; - - int seconds = Convert.ToInt32(timeStamp.Subtract(new DateTime(1970, 1, 1, 0, 0, 0, 0)).TotalSeconds); - string path = String.Format("D:/rootcause/Dataset_yaniv/raw_data_201908_202002/{0}.csv", seconds); - List points = GetPoints(path); - List slices = new List(); - slices.Add(new MetricSlice(timeStamp, points)); - - PredictionEngine engine = GetRootCausePredictionEngine(); - - var newRootCauseInput = new RootCauseLocalizationData(timeStamp, rootNodeMap[timeStamp], new List() { new MetricSlice(timeStamp, points) }, DTRootCauseLocalizationEstimator.AggregateType.Sum, aggSymbol); - - List list = new List(); - GetRootCause(list, newRootCauseInput, engine); - - List> labeledRootCause = labeledRootCauseMap[timeStamp]; - List> detectedRootCause = ConvertRootCauseItemToDic(list); - RemoveAggSymbol(detectedRootCause, aggSymbol); - - Tuple evaluation = ScoreRootCause(detectedRootCause, labeledRootCause, exactly, timeStamp); - totalTp += evaluation.Item1; - totalFp += evaluation.Item2; - totalFn += evaluation.Item3; - totalCount++; - } - - double precision = (double)totalTp / (totalTp + totalFp); - double recall = (double)totalTp / (totalTp + totalFn); - double f1 = 2 * precision * recall / (precision + recall); - Console.WriteLine(String.Format("Total Count : {0}, TP: {1}, FP: {2}, FN: {3}", totalCount, totalTp, totalFp, totalFn)); - Console.WriteLine(String.Format("Precision : {0}, Recall: {1}, F1: {2}", precision, recall, f1)); - } - - private static Tuple ScoreRootCause(List> detectedRootCause, List> labeledRootCause, bool exactly, DateTime timeStamp) - { - int tp = 0; - int fp = 0; - int fn; ; - List labelSet = new List(); - foreach (Dictionary cause in detectedRootCause) - { - string tpCause = FindTruePositive(cause, labeledRootCause, exactly); - if (tpCause == null) - { - fp++; - Console.WriteLine(String.Format("FP : timestamp - {0}, detected root cause ", timeStamp)); - Console.WriteLine(string.Join(Environment.NewLine, cause)); - Console.WriteLine(" "); - } - else - { - tp++; - labelSet.Add(tpCause); - } - } - - fn = labeledRootCause.Count - labelSet.Count; - if (fn != 0) - { - List> nCause = GetNegtiveCause(labeledRootCause, labelSet); - if (nCause.Count > 0) - { - Console.WriteLine(String.Format("FN : timestamp - {0}", timeStamp)); - foreach (Dictionary cause in nCause) - { - Console.WriteLine(string.Join(Environment.NewLine, cause)); - Console.WriteLine("---------------------"); - } - - } - } - - return new Tuple(tp, fp, fn); - } - - private static List> GetNegtiveCause(List> labelCauses, List labelSet) - { - List> causeList = new List>(); - foreach (Dictionary cause in labelCauses) - { - if (!labelSet.Contains(GetDicHashCode(cause))) - { - causeList.Add(cause); - } - } - return causeList; - } - - private static string FindTruePositive(Dictionary cause, List> labelCauses, bool exactly) - { - foreach (Dictionary label in labelCauses) - { - string id = GetDicHashCode(label); - int compare = CompareCause(cause, label); - if (compare == 0) - { - return id; - } - else if (!exactly && (compare == 1 || compare == 2)) - { - return id; - } - } - return null; - } - - - private static string GetDicHashCode(Dictionary dic) - { - return dic.GetHashCode().ToString(); - } - - private static int CompareCause(Dictionary detect, Dictionary label) - { - - if (detect.Equals(label)) - { - return 0; - } - else if (DTRootCauseLocalizationUtils.ContainsAll(detect, label)) - { - return 1; - } - else if (DTRootCauseLocalizationUtils.ContainsAll(label, detect)) - { - return 2; - } - return 3; - } - private static List> ConvertRootCauseItemToDic(List items) - { - List> list = new List>(); - foreach (RootCauseItem item in items) - { - list.Add(item.RootCause); - } - return list; - } - - private static void RemoveAggSymbol(List> dimensions, string aggSymbol) - { - foreach (Dictionary dim in dimensions) - { - foreach (string key in dim.Keys) - { - if (dim[key].Equals(aggSymbol)) - { - dim.Remove(key); - } - } - } - } - - private static PredictionEngine GetRootCausePredictionEngine() - { - //// Create an root cause localizatiom input list from csv. - var rootCauseLocalizationData = new List() { new RootCauseLocalizationData(new DateTime(), new Dictionary(), new List() { new MetricSlice(new DateTime(), new List()) }, DTRootCauseLocalizationEstimator.AggregateType.Sum, "SUM") }; - - - var ml = new MLContext(1); - // Convert the list of root cause data to an IDataView object, which is consumable by ML.NET API. - var data = ml.Data.LoadFromEnumerable(rootCauseLocalizationData); - - // Create pipeline to localize root cause by decision tree. - var pipeline = ml.Transforms.LocalizeRootCauseByDT(nameof(RootCauseLocalizationTransformedData.RootCause), nameof(RootCauseLocalizationData.Input)); - - // Fit the model. - var model = pipeline.Fit(data); - - // Test path: input list -> IDataView -> Enumerable of RootCauseLocalizationInputs. - var transformedData = model.Transform(data); - - // Load input list in DataView back to Enumerable. - var transformedDataPoints = ml.Data.CreateEnumerable(transformedData, false); - - var engine = ml.Model.CreatePredictionEngine(model); - return engine; - } - - private static string _ocsDataCenter = "OCSDatacenter"; - private static string _appType = "AppType"; - private static string _releaseAudienceGroup = "Release_AudienceGroup"; - private static string _wacDatacenter = "WACDatacenter"; - private static string _requestType = "RequestType"; - private static string _statusCode = "StatusCode"; - - private static List _dimensionKeys = new List() { _ocsDataCenter, _appType, _releaseAudienceGroup, _wacDatacenter, _statusCode, _requestType }; - - private static Dictionary> GetAnomalyRootMap() - { - var anomalyRootData = GetDataTabletFromCSVFile("D:/rootcause/Dataset_yaniv/root_cause_201908_202002/anomaly_root.csv"); - - Dictionary> rootNodeMap = new Dictionary>(); - foreach (DataRow row in anomalyRootData.Rows) - { - // load the data, build the RootCauseInput, take care of empty value - long seconds = long.Parse(row["TimeStamp"].ToString()); - DateTime t = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); - - Dictionary dimension = new Dictionary(); - foreach (string key in _dimensionKeys) - { - if (!row[key].ToString().Equals("")) - { - dimension.Add(key, row[key].ToString()); - } - } - - rootNodeMap.Add(t, dimension); - } - return rootNodeMap; - } - - private static Dictionary>> GetLabeledRootCauseMap() - { - var labeldRootCause = GetDataTabletFromCSVFile("D:/rootcause/Dataset_yaniv/root_cause_201908_202002/labeled_root_cause.csv"); - - Dictionary>> map = new Dictionary>>(); - foreach (DataRow row in labeldRootCause.Rows) - { - // load the data, build the labled result, take care of empty value - long seconds = long.Parse(row["TimeStamp"].ToString()); - DateTime t = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); - - Dictionary dimension = new Dictionary(); - foreach (string key in _dimensionKeys) - { - if (!row[key].ToString().Equals("")) - { - dimension.Add(key, row[key].ToString()); - } - } - - if (map.ContainsKey(t)) - { - map[t].Add(dimension); - } - else - { - map.Add(t, new List>() { dimension }); - } - } - return map; - } - - private static List GetPoints(string path) - { - - - var inputData = GetDataTabletFromCSVFile(path); - - DateTime timeStamp = new DateTime(); - - List points = new List(); - foreach (DataRow row in inputData.Rows) - { - // load the data, build the RootCauseInput, take care of empty value - long seconds = long.Parse(row["TimeStamp"].ToString()); - timeStamp = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); - double value = Double.Parse(row["Value"].ToString()); - double expectedValue = 0; - if (!row["ExpectedValue"].ToString().Equals("")) - { - expectedValue = Double.Parse(row["ExpectedValue"].ToString()); - } - bool isAnomaly = Boolean.Parse(row["IsAnomaly"].ToString()); - Dictionary dimension = new Dictionary(); - foreach (string key in _dimensionKeys) - { - if (!row[key].ToString().Equals("")) - { - dimension.Add(key, row[key].ToString()); - } - } - - points.Add(new Microsoft.ML.TimeSeries.Point(value, expectedValue, isAnomaly, dimension)); ; - } - - return points; - } - - private static void GetRootCause(List rootCauseList, RootCauseLocalizationData inputData, PredictionEngine engine) - { - - RootCauseLocalizationTransformedData incrementalResult = engine.Predict(inputData); - - if (incrementalResult.RootCause.Items.Count == 0 || ( - incrementalResult.RootCause.Items.Count == 1 && incrementalResult.RootCause.Items[0].RootCause.Equals(inputData.Input.AnomalyDimensions) - )) - { - if (!rootCauseList.Contains(new RootCauseItem(inputData.Input.AnomalyDimensions))) - { - rootCauseList.Add(new RootCauseItem(inputData.Input.AnomalyDimensions)); - - } - return; - } - else - { - foreach (RootCauseItem item in incrementalResult.RootCause.Items) - { - RootCauseLocalizationData newData = new RootCauseLocalizationData(inputData.Input.AnomalyTimestamp, - item.RootCause, inputData.Input.Slices, inputData.Input.AggType, inputData.Input.AggSymbol); - GetRootCause(rootCauseList, newData, engine); - } - } - } - - private static DataTable GetDataTabletFromCSVFile(string filePath) - { - DataTable csvData = new DataTable(); - - - using (TextFieldParser csvReader = new TextFieldParser(filePath)) - { - csvReader.SetDelimiters(new string[] { "," }); - csvReader.HasFieldsEnclosedInQuotes = true; - string[] colFields = csvReader.ReadFields(); - foreach (string column in colFields) - { - DataColumn datecolumn = new DataColumn(column); - datecolumn.AllowDBNull = true; - csvData.Columns.Add(datecolumn); - } - - while (!csvReader.EndOfData) - { - string[] fieldData = csvReader.ReadFields(); - //Making empty value as null - for (int i = 0; i < fieldData.Length; i++) - { - if (fieldData[i] == "") - { - fieldData[i] = null; - } - } - csvData.Rows.Add(fieldData); - } - } - - return csvData; - } - - private class RootCauseLocalizationData - { - [RootCauseLocalizationInputType] - public RootCauseLocalizationInput Input { get; set; } - - public RootCauseLocalizationData() - { - Input = null; - } - - public RootCauseLocalizationData(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, DTRootCauseLocalizationEstimator.AggregateType aggregateteType, string aggregateSymbol) - { - Input = new RootCauseLocalizationInput(anomalyTimestamp, anomalyDimensions, slices, aggregateteType, aggregateSymbol); - } - } - - private class RootCauseLocalizationTransformedData - { - [RootCauseType()] - public RootCause RootCause { get; set; } - - public RootCauseLocalizationTransformedData() - { - RootCause = null; - } - } - } -} From eadada42acd25ec6396169f8a547bfeddf43a4ed Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:41:29 +0800 Subject: [PATCH 07/36] merge --- test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index fd853e39d5..77260ec7c4 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -8,6 +8,7 @@ using Microsoft.ML.Data; using Microsoft.ML.TestFramework; using Microsoft.ML.TimeSeries; +using Microsoft.ML.TimeSeries; using Microsoft.ML.Transforms.TimeSeries; using Xunit; using Xunit.Abstractions; From 6cf9b596b49d34c27c2e033c11461498360ab466 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:43:58 +0800 Subject: [PATCH 08/36] merge --- docs/samples/Microsoft.ML.Samples/Program.cs | 1 + .../DTRootCauseLocalizationType.cs | 168 ++++++++++++++++++ .../TimeSeriesDirectApi.cs | 30 +++- 3 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.ML.TimeSeries/DTRootCauseLocalizationType.cs diff --git a/docs/samples/Microsoft.ML.Samples/Program.cs b/docs/samples/Microsoft.ML.Samples/Program.cs index a3da984b16..930ccc57f3 100644 --- a/docs/samples/Microsoft.ML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.Samples/Program.cs @@ -14,6 +14,7 @@ internal static void RunAll() foreach (var type in Assembly.GetExecutingAssembly().GetTypes()) { if (type.Name.Equals("LocalizeRootCauseEvaluation")) + //if (type.Name.Equals("LocalizeRootCauseByDT")) { var sample = type.GetMethod("Example", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); diff --git a/src/Microsoft.ML.TimeSeries/DTRootCauseLocalizationType.cs b/src/Microsoft.ML.TimeSeries/DTRootCauseLocalizationType.cs new file mode 100644 index 0000000000..1e5e3f881e --- /dev/null +++ b/src/Microsoft.ML.TimeSeries/DTRootCauseLocalizationType.cs @@ -0,0 +1,168 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using Microsoft.ML.Data; + +namespace Microsoft.ML.TimeSeries +{ + /// + /// Allows a member to be marked as a , primarily allowing one to set + /// root cause localization input. + /// + public sealed class RootCauseLocalizationInputTypeAttribute : DataViewTypeAttribute + { + /// + /// Create a root cause localizagin input type. + /// + public RootCauseLocalizationInputTypeAttribute() + { + } + + /// + /// Equal function. + /// + public override bool Equals(DataViewTypeAttribute other) + { + if (!(other is RootCauseLocalizationInputTypeAttribute otherAttribute)) + return false; + return true; + } + + /// + /// Produce the same hash code for all RootCauseLocalizationInputTypeAttribute. + /// + public override int GetHashCode() + { + return 0; + } + + public override void Register() + { + DataViewTypeManager.Register(new RootCauseLocalizationInputDataViewType(), typeof(RootCauseLocalizationInput), this); + } + } + + /// + /// Allows a member to be marked as a , primarily allowing one to set + /// root cause result. + /// + public sealed class RootCauseTypeAttribute : DataViewTypeAttribute + { + /// + /// Create an root cause type. + /// + public RootCauseTypeAttribute() + { + } + + /// + /// RootCauseTypeAttribute with the same type should equal. + /// + public override bool Equals(DataViewTypeAttribute other) + { + if (other is RootCauseTypeAttribute otherAttribute) + return true; + return false; + } + + /// + /// Produce the same hash code for all RootCauseTypeAttribute. + /// + public override int GetHashCode() + { + return 0; + } + + public override void Register() + { + DataViewTypeManager.Register(new RootCauseDataViewType(), typeof(RootCause), this); + } + } + + public sealed class RootCause + { + public List Items { get; set; } + } + + public sealed class RootCauseLocalizationInput + { + public DateTime AnomalyTimestamp { get; set; } + + public Dictionary AnomalyDimensions { get; set; } + + public List Slices { get; set; } + + public AggregateType AggType { get; set; } + + public string AggSymbol { get; set; } + + public RootCauseLocalizationInput(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, AggregateType aggregateType, string aggregateSymbol) + { + AnomalyTimestamp = anomalyTimestamp; + AnomalyDimensions = anomalyDimensions; + Slices = slices; + AggType = aggregateType; + AggSymbol = aggregateSymbol; + } + public void Dispose() + { + AnomalyDimensions = null; + Slices = null; + } + } + + public sealed class RootCauseDataViewType : StructuredDataViewType + { + public RootCauseDataViewType() + : base(typeof(RootCause)) + { + } + + public override bool Equals(DataViewType other) + { + if (other == this) + return true; + if (!(other is RootCauseDataViewType tmp)) + return false; + return true; + } + + public override int GetHashCode() + { + return 0; + } + + public override string ToString() + { + return typeof(RootCauseDataViewType).Name; + } + } + + public sealed class RootCauseLocalizationInputDataViewType : StructuredDataViewType + { + public RootCauseLocalizationInputDataViewType() + : base(typeof(RootCauseLocalizationInput)) + { + } + + public override bool Equals(DataViewType other) + { + if (!(other is RootCauseLocalizationInputDataViewType tmp)) + return false; + return true; + } + + public override int GetHashCode() + { + return 0; + } + + public override string ToString() + { + return typeof(RootCauseLocalizationInputDataViewType).Name; + } + } +} diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index 77260ec7c4..642cd1d125 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -665,7 +665,6 @@ public void RootCauseLocalization() { // Create an root cause localizatiom input var rootCauseLocalizationInput = new RootCauseLocalizationInput(GetRootCauseTimestamp(), GetRootCauseAnomalyDimension(), new List() { new MetricSlice(GetRootCauseTimestamp(), GetRootCauseLocalizationPoints()) }, AggregateType.Sum, _rootCauseAggSymbol); - var ml = new MLContext(1); RootCause rootCause = ml.AnomalyDetection.LocalizeRootCause(rootCauseLocalizationInput); Assert.NotNull(rootCause); @@ -682,6 +681,16 @@ public void RootCauseLocalization() { Assert.Equal(expectedDim[pair.Key], pair.Value); } + var newRootCauseInput = new RootCauseLocalizationData(timeStamp, GetAnomalyDimension(), new List() { new MetricSlice(timeStamp, GetRootCauseLocalizationPoints()) }, AggregateType.Sum, _aggSymbol); + + Dictionary expectedDim = new Dictionary(); + expectedDim.Add("Country","UK"); + expectedDim.Add("DeviceType",_aggSymbol); + expectedDim.Add("DataCenter","DC1"); + + foreach (KeyValuePair pair in transformedRootCause.RootCause.Items[0].Dimension) { + Assert.Equal(expectedDim[pair.Key], pair.Value); + } } private static List GetRootCauseLocalizationPoints() { @@ -742,5 +751,24 @@ private static Dictionary GetRootCauseAnomalyDimension() private static DateTime GetRootCauseTimestamp() { return new DateTime(2020, 3, 23, 0, 0, 0); + + return points; + } + + private static Dictionary GetAnomalyDimension() + { + Dictionary dim = new Dictionary(); + dim.Add("Country", "UK"); + dim.Add("DeviceType", _aggSymbol); + dim.Add("DataCenter", _aggSymbol); + + return dim; + } + + private static DateTime GetCurrentTimestamp() + { + return new DateTime(); + } + } } From 36d1625efc80a6375055ba88de2d083abf245d1a Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:45:20 +0800 Subject: [PATCH 09/36] merge --- .../TimeSeries/LocalizeRootCauseEvaluation.cs | 409 ++++++++ .../DTRootCauseAnalyzer.cs | 905 ++++++++++++++++++ .../TimeSeriesDirectApi.cs | 85 ++ 3 files changed, 1399 insertions(+) create mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs create mode 100644 src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs new file mode 100644 index 0000000000..47163a49dc --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs @@ -0,0 +1,409 @@ +using System; +using System.Collections.Generic; +using System.Data; +using Microsoft.ML; +using Microsoft.ML.TimeSeries; +using Microsoft.ML.Transforms.TimeSeries; + +using Microsoft.VisualBasic.FileIO; + +namespace Samples.Dynamic.Transforms.TimeSeries +{ + public static class LocalizeRootCauseEvaluation + { + public static void Example() + { + Dictionary> rootNodeMap = GetAnomalyRootMap(); + Dictionary>> labeledRootCauseMap = GetLabeledRootCauseMap(); + + string aggSymbol = "##EMPTY##awqegp##"; + + int totalTp = 0; + int totalFp = 0; + int totalFn = 0; + int totalCount = 0; + + bool exactly = false; + + int totalRunTime = 0; + + foreach (KeyValuePair> item in rootNodeMap) + { + DateTime timeStamp = item.Key; + + DateTime filterTime = DateTime.ParseExact("2019-11-13 13:00:00,000", "yyyy-MM-dd HH:mm:ss,fff", + System.Globalization.CultureInfo.InvariantCulture); + + //if (timeStamp.CompareTo(filterTime).Equals(0)) + { + int seconds = Convert.ToInt32(timeStamp.Subtract(new DateTime(1970, 1, 1, 0, 0, 0, 0)).TotalSeconds); + string path = String.Format("D:/rootcause/Dataset_yaniv/raw_data_201908_202002/{0}.csv", seconds); + List points = GetPoints(path); + List slices = new List(); + slices.Add(new MetricSlice(timeStamp, points)); + + PredictionEngine engine = GetRootCausePredictionEngine(); + + var newRootCauseInput = new RootCauseLocalizationData(timeStamp, rootNodeMap[timeStamp], new List() { new MetricSlice(timeStamp, points) }, AggregateType.Sum, aggSymbol); + + List list = new List(); + int startTime = System.Environment.TickCount; + GetRootCause(list, newRootCauseInput, engine); + int endTime = System.Environment.TickCount; + int runTime = endTime - startTime; + totalRunTime += runTime; + + List> labeledRootCause = labeledRootCauseMap[timeStamp]; + List> detectedRootCause = ConvertRootCauseItemToDic(list); + RemoveAggSymbol(detectedRootCause, aggSymbol); + + Tuple evaluation = EvaluateRootCauseResult(detectedRootCause, labeledRootCause, exactly, timeStamp); + totalTp += evaluation.Item1; + totalFp += evaluation.Item2; + totalFn += evaluation.Item3; + totalCount++; + } + } + + double precision = (double)totalTp / (totalTp + totalFp); + double recall = (double)totalTp / (totalTp + totalFn); + double f1 = 2 * precision * recall / (precision + recall); + Console.WriteLine(String.Format("Total Count : {0}, TP: {1}, FP: {2}, FN: {3}", totalCount, totalTp, totalFp, totalFn)); + Console.WriteLine(String.Format("Precision : {0}, Recall: {1}, F1: {2}", precision, recall, f1)); + Console.WriteLine(String.Format("Mean calculation time is : {0} ms", (double)totalRunTime / totalCount)); + } + + private static Tuple EvaluateRootCauseResult(List> detectedRootCause, List> labeledRootCause, bool exactly, DateTime timeStamp) + { + int tp = 0; + int fp = 0; + int fn; + List labelSet = new List(); + foreach (Dictionary cause in detectedRootCause) + { + string tpCause = FindTruePositive(cause, labeledRootCause, exactly); + if (tpCause == null) + { + fp++; + Console.WriteLine(String.Format("FP : timestamp - {0}, detected root cause ", timeStamp)); + Console.WriteLine(string.Join(Environment.NewLine, cause)); + Console.WriteLine(" "); + } + else + { + tp++; + labelSet.Add(tpCause); + } + } + + fn = labeledRootCause.Count - labelSet.Count; + if (fn != 0) + { + List> nCause = GetFNegtiveCause(labeledRootCause, labelSet); + if (nCause.Count > 0) + { + Console.WriteLine(String.Format("FN : timestamp - {0}, labeled root cause", timeStamp)); + foreach (Dictionary cause in nCause) + { + Console.WriteLine(string.Join(Environment.NewLine, cause)); + Console.WriteLine("---------------------"); + } + + } + } + + return new Tuple(tp, fp, fn); + } + + private static List> GetFNegtiveCause(List> labelCauses, List labelSet) + { + List> causeList = new List>(); + foreach (Dictionary cause in labelCauses) + { + if (!labelSet.Contains(GetDicHashCode(cause))) + { + causeList.Add(cause); + } + } + return causeList; + } + + private static string FindTruePositive(Dictionary cause, List> labelCauses, bool exactly) + { + foreach (Dictionary label in labelCauses) + { + string id = GetDicHashCode(label); + int compare = CompareCause(cause, label); + if (compare == 0) + { + return id; + } + else if (!exactly && (compare == 1 || compare == 2)) + { + return id; + } + } + return null; + } + + + private static string GetDicHashCode(Dictionary dic) + { + return dic.GetHashCode().ToString(); + } + + private static int CompareCause(Dictionary detect, Dictionary label) + { + if (detect.Equals(label)) + { + return 0; + } + else if (DTRootCauseAnalyzer.ContainsAll(detect, label)) + { + return 1; + } + else if (DTRootCauseAnalyzer.ContainsAll(label, detect)) + { + return 2; + } + return 3; + } + private static List> ConvertRootCauseItemToDic(List items) + { + List> list = new List>(); + foreach (RootCauseItem item in items) + { + list.Add(item.Dimension); + } + return list; + } + + private static void RemoveAggSymbol(List> dimensions, string aggSymbol) + { + foreach (Dictionary dim in dimensions) + { + foreach (string key in dim.Keys) + { + if (dim[key].Equals(aggSymbol)) + { + dim.Remove(key); + } + } + } + } + + private static PredictionEngine GetRootCausePredictionEngine() + { + //// Create an root cause localizatiom input list from csv. + var rootCauseLocalizationData = new List() { new RootCauseLocalizationData(new DateTime(), new Dictionary(), new List() { new MetricSlice(new DateTime(), new List()) }, AggregateType.Sum, "SUM") }; + + + var ml = new MLContext(1); + // Convert the list of root cause data to an IDataView object, which is consumable by ML.NET API. + var data = ml.Data.LoadFromEnumerable(rootCauseLocalizationData); + + // Create pipeline to localize root cause by decision tree. + var pipeline = ml.Transforms.LocalizeRootCauseByDT(nameof(RootCauseLocalizationTransformedData.RootCause), nameof(RootCauseLocalizationData.Input)); + + // Fit the model. + var model = pipeline.Fit(data); + + // Test path: input list -> IDataView -> Enumerable of RootCauseLocalizationInputs. + var transformedData = model.Transform(data); + + // Load input list in DataView back to Enumerable. + var transformedDataPoints = ml.Data.CreateEnumerable(transformedData, false); + + var engine = ml.Model.CreatePredictionEngine(model); + return engine; + } + + private static string _ocsDataCenter = "OCSDatacenter"; + private static string _appType = "AppType"; + private static string _releaseAudienceGroup = "Release_AudienceGroup"; + private static string _wacDatacenter = "WACDatacenter"; + private static string _requestType = "RequestType"; + private static string _statusCode = "StatusCode"; + + private static List _dimensionKeys = new List() { _ocsDataCenter, _appType, _releaseAudienceGroup, _wacDatacenter, _statusCode, _requestType }; + + private static Dictionary> GetAnomalyRootMap() + { + var anomalyRootData = GetDataTabletFromCSVFile("D:/rootcause/Dataset_yaniv/root_cause_201908_202002/anomaly_root.csv"); + + Dictionary> rootNodeMap = new Dictionary>(); + foreach (DataRow row in anomalyRootData.Rows) + { + // load the data, build the RootCauseInput, take care of empty value + long seconds = long.Parse(row["TimeStamp"].ToString()); + DateTime t = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); + + Dictionary dimension = new Dictionary(); + foreach (string key in _dimensionKeys) + { + if (!row[key].ToString().Equals("")) + { + dimension.Add(key, row[key].ToString()); + } + } + + rootNodeMap.Add(t, dimension); + } + return rootNodeMap; + } + + private static Dictionary>> GetLabeledRootCauseMap() + { + var labeldRootCause = GetDataTabletFromCSVFile("D:/rootcause/Dataset_yaniv/root_cause_201908_202002/labeled_root_cause.csv"); + + Dictionary>> map = new Dictionary>>(); + foreach (DataRow row in labeldRootCause.Rows) + { + // load the data, build the labled result, take care of empty value + long seconds = long.Parse(row["TimeStamp"].ToString()); + DateTime t = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); + + Dictionary dimension = new Dictionary(); + foreach (string key in _dimensionKeys) + { + if (!row[key].ToString().Equals("")) + { + dimension.Add(key, row[key].ToString()); + } + } + + if (map.ContainsKey(t)) + { + map[t].Add(dimension); + } + else + { + map.Add(t, new List>() { dimension }); + } + } + return map; + } + + private static List GetPoints(string path) + { + var inputData = GetDataTabletFromCSVFile(path); + + DateTime timeStamp = new DateTime(); + + List points = new List(); + foreach (DataRow row in inputData.Rows) + { + // load the data, build the RootCauseInput, take care of empty value + long seconds = long.Parse(row["TimeStamp"].ToString()); + timeStamp = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); + double value = Double.Parse(row["Value"].ToString()); + double expectedValue = 0; + if (!row["ExpectedValue"].ToString().Equals("")) + { + expectedValue = Double.Parse(row["ExpectedValue"].ToString()); + } + bool isAnomaly = Boolean.Parse(row["IsAnomaly"].ToString()); + Dictionary dimension = new Dictionary(); + foreach (string key in _dimensionKeys) + { + if (!row[key].ToString().Equals("")) + { + dimension.Add(key, row[key].ToString()); + } + } + + points.Add(new Microsoft.ML.TimeSeries.Point(value, expectedValue, isAnomaly, dimension)); ; + } + + return points; + } + + private static void GetRootCause(List rootCauseList, RootCauseLocalizationData inputData, PredictionEngine engine) + { + RootCauseLocalizationTransformedData incrementalResult = engine.Predict(inputData); + + if (incrementalResult.RootCause.Items.Count == 0 || ( + incrementalResult.RootCause.Items.Count == 1 && incrementalResult.RootCause.Items[0].Dimension.Equals(inputData.Input.AnomalyDimensions) + )) + { + if (!rootCauseList.Contains(new RootCauseItem(inputData.Input.AnomalyDimensions))) + { + rootCauseList.Add(new RootCauseItem(inputData.Input.AnomalyDimensions)); + + } + return; + } + else + { + foreach (RootCauseItem item in incrementalResult.RootCause.Items) + { + RootCauseLocalizationData newData = new RootCauseLocalizationData(inputData.Input.AnomalyTimestamp, + item.Dimension, inputData.Input.Slices, inputData.Input.AggType, inputData.Input.AggSymbol); + GetRootCause(rootCauseList, newData, engine); + } + } + } + + private static DataTable GetDataTabletFromCSVFile(string filePath) + { + DataTable csvData = new DataTable(); + + + using (TextFieldParser csvReader = new TextFieldParser(filePath)) + { + csvReader.SetDelimiters(new string[] { "," }); + csvReader.HasFieldsEnclosedInQuotes = true; + string[] colFields = csvReader.ReadFields(); + foreach (string column in colFields) + { + DataColumn datecolumn = new DataColumn(column); + datecolumn.AllowDBNull = true; + csvData.Columns.Add(datecolumn); + } + + while (!csvReader.EndOfData) + { + string[] fieldData = csvReader.ReadFields(); + //Making empty value as null + for (int i = 0; i < fieldData.Length; i++) + { + if (fieldData[i] == "") + { + fieldData[i] = null; + } + } + csvData.Rows.Add(fieldData); + } + } + + return csvData; + } + + private class RootCauseLocalizationData + { + [RootCauseLocalizationInputType] + public RootCauseLocalizationInput Input { get; set; } + + public RootCauseLocalizationData() + { + Input = null; + } + + public RootCauseLocalizationData(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, AggregateType aggregateteType, string aggregateSymbol) + { + Input = new RootCauseLocalizationInput(anomalyTimestamp, anomalyDimensions, slices, aggregateteType, aggregateSymbol); + } + } + + private class RootCauseLocalizationTransformedData + { + [RootCauseType()] + public RootCause RootCause { get; set; } + + public RootCauseLocalizationTransformedData() + { + RootCause = null; + } + } + } +} diff --git a/src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs new file mode 100644 index 0000000000..b7a66582b3 --- /dev/null +++ b/src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs @@ -0,0 +1,905 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.ML.Internal.Utilities; + +namespace Microsoft.ML.TimeSeries +{ + public class DTRootCauseAnalyzer + { + public static List GetTotalPointsForAnomalyTimestamp(RootCauseLocalizationInput src, Dictionary subDim) + { + List points = new List(); + foreach (MetricSlice slice in src.Slices) + { + if (slice.TimeStamp.Equals(src.AnomalyTimestamp)) + { + points = slice.Points; + } + } + + List totalPoints = DTRootCauseAnalyzer.SelectPoints(points, subDim); + + return totalPoints; + } + + public static void GetRootCauseList(RootCauseLocalizationInput src, ref RootCause dst, DimensionInfo dimensionInfo, List totalPoints, Dictionary subDim) + { + PointTree pointTree = DTRootCauseAnalyzer.BuildPointTree(totalPoints, dimensionInfo.AggDim, subDim, src.AggSymbol, src.AggType); + PointTree anomalyTree = DTRootCauseAnalyzer.BuildPointTree(totalPoints, dimensionInfo.AggDim, subDim, src.AggSymbol, src.AggType, true); + + //which means there is no aggregation in the input anomaly dimension + if (anomalyTree.ParentNode == null) + { + return; + } + + List rootCauses = new List(); + // no point under anomaly dimension + if (totalPoints.Count == 0) + { + if (anomalyTree.Leaves.Count != 0) + { + throw new Exception("point leaves not match with anomaly leaves"); + } + + rootCauses.Add(new RootCauseItem(src.AnomalyDimensions)); + } + else + { + double totalEntropy = 1; + if (anomalyTree.Leaves.Count > 0) + { + totalEntropy = DTRootCauseAnalyzer.GetEntropy(pointTree.Leaves.Count, anomalyTree.Leaves.Count); + } + + if (totalEntropy > 0.9) + { + rootCauses.AddRange(DTRootCauseAnalyzer.LocalizeRootCauseByDimension(pointTree.Leaves, anomalyTree, pointTree, totalEntropy, src.AnomalyDimensions)); + } + else + { + rootCauses.AddRange(DTRootCauseAnalyzer.LocalizeRootCauseByDimension(pointTree.Leaves, anomalyTree, pointTree, totalEntropy, src.AnomalyDimensions)); + } + + dst.Items = rootCauses; + } + } + + public static DimensionInfo SeperateDimension(Dictionary dimensions, string aggSymbol) + { + DimensionInfo info = DimensionInfo.CreateDefaultInstance(); + foreach (KeyValuePair entry in dimensions) + { + string key = entry.Key; + if (aggSymbol.Equals(entry.Value)) + { + info.AggDim.Add(key); + } + else + { + info.DetailDim.Add(key); + } + } + + return info; + } + + protected static PointTree BuildPointTree(List pointList, List aggDims, Dictionary subDim, string aggSymbol, AggregateType aggType, bool filterByAnomaly = false) + { + PointTree tree = PointTree.CreateDefaultInstance(); + + foreach (Point point in pointList) + { + bool isValidPoint = true; + if (filterByAnomaly) + { + isValidPoint = point.IsAnomaly == true; + } + if (ContainsAll(point.Dimensions, subDim) && isValidPoint) + { + if (aggDims.Count == 0) + { + tree.ParentNode = point; + tree.Leaves.Add(point); + } + else + { + int aggNum = 0; + string nextDim = null; + + foreach (string dim in aggDims) + { + if (IsAggregationDimension(point.Dimensions[dim], aggSymbol)) + { + aggNum++; + } + else + { + nextDim = dim; + } + } + + if (aggNum == aggDims.Count) + { + tree.ParentNode = point; + } + else if (aggNum == aggDims.Count - 1) + { + if (!tree.ChildrenNodes.ContainsKey(nextDim)) + { + tree.ChildrenNodes.Add(nextDim, new List()); + } + tree.ChildrenNodes[nextDim].Add(point); + } + + if (aggNum == 0) + { + tree.Leaves.Add(point); + } + } + } + } + + return tree; + } + private static PointTree CompleteTreeBottomUp(PointTree tree, AggregateType aggType, string aggSymbol, List aggDims) + { + + if (tree.Leaves.Count == 0) return tree; + + Dictionary> map = new Dictionary>(); + foreach (Point p in tree.Leaves) + { + foreach (KeyValuePair keyValuePair in p.Dimensions) + { + if (aggDims.Contains(keyValuePair.Key)) + { + if (map.ContainsKey(keyValuePair.Key)) + { + map[keyValuePair.Key].Add(keyValuePair.Value); + } + else + { + map.Add(keyValuePair.Key, new HashSet() { keyValuePair.Value }); + } + } + } + } + + foreach (KeyValuePair> pair in map) + { + if (tree.ChildrenNodes.ContainsKey(pair.Key)) + { + if (tree.ChildrenNodes[pair.Key].Count < pair.Value.Count) + { + foreach (string value in pair.Value) + { + if (!IsAggDimensionExisted(pair.Key, value, tree.ChildrenNodes[pair.Key])) + { + Point p = SimulateBottomUpValue(tree.Leaves, pair.Key, value, aggType, aggSymbol); + tree.ChildrenNodes[pair.Key].Add(p); + } + } + } + } + else + { + List childPoints = new List(); + foreach (string value in pair.Value) + { + //simulate the aggregation value + Point p = SimulateBottomUpValue(tree.Leaves, pair.Key, value, aggType, aggSymbol); + childPoints.Add(p); + } + + tree.ChildrenNodes.Add(pair.Key, childPoints); + } + } + + return tree; + } + + private static bool IsAggDimensionExisted(string key, string value, List points) + { + foreach (Point p in points) + { + if (p.Dimensions[key].Equals(value)) + { + return true; + } + } + return false; + } + + private static Point SimulateBottomUpValue(List leaves, string key, string keyValue, AggregateType type, string aggSymbol) + { + Point p = null; + + Dictionary dimension = new Dictionary(); + + dimension.Add(key, keyValue); + + foreach (KeyValuePair pair in leaves[0].Dimensions) + { + if (!pair.Key.Equals(key)) + { + dimension.Add(pair.Key, aggSymbol); + } + } + + if (type.Equals(AggregateType.Sum)) + { + + bool isAnomaly = false; + double value = 0; + double expectedValue = 0; + foreach (Point leave in leaves) + { + + if (leave.Dimensions.ContainsKey(key) && leave.Dimensions[key].Equals(keyValue)) + { + value += leave.Value; + expectedValue = leave.ExpectedValue; + isAnomaly = isAnomaly || leave.IsAnomaly; + } + } + + p = new Point(value, expectedValue, isAnomaly, dimension); + } + + return p; + } + + public static Dictionary GetSubDim(Dictionary dimension, List keyList) + { + Dictionary subDim = new Dictionary(); + + foreach (String dim in keyList) + { + subDim.Add(dim, dimension[dim]); + } + return subDim; + } + + protected static List SelectPoints(List points, Dictionary subDim) + { + List list = new List(); + + foreach (Point point in points) + { + if (ContainsAll(point.Dimensions, subDim)) + { + //remove duplicated points + if (!list.Contains(point)) + { + list.Add(point); + } + } + } + + return list; + } + + protected static List LocalizeRootCauseByDimension(List totalPoints, PointTree anomalyTree, PointTree pointTree, double totoalEntropy, Dictionary anomalyDimension) + { + var set = anomalyTree.ChildrenNodes.Keys; + + BestDimension best = null; + if (anomalyTree.Leaves.Count > 0) + { + best = SelectBestDimension(totalPoints, anomalyTree.Leaves, set.ToList(), totoalEntropy); + } + else + { + //has no leaves information, should calculate the entropy information according to the children nodes + best = SelectBestDimension(pointTree.ChildrenNodes, anomalyTree.ChildrenNodes, set.ToList(), totoalEntropy); + } + + if (best == null) + { + return new List() { new RootCauseItem(anomalyDimension) }; + } + + List children = GetTopAnomaly(anomalyTree.ChildrenNodes[best.DimensionKey], anomalyTree.ParentNode, totalPoints, best.DimensionKey); + if (children == null) + { + //As the cause couldn't be found, the root cause should be itself + return new List() { new RootCauseItem(anomalyDimension, best.DimensionKey) }; + } + else + { + List causes = new List(); + // For the found causes, we return the result + foreach (Point anomaly in children) + { + causes.Add(new RootCauseItem(UpdateDimensionValue(anomalyDimension, best.DimensionKey, anomaly.Dimensions[best.DimensionKey]), best.DimensionKey)); + } + return causes; + } + } + + protected static double GetEntropy(int totalNum, int anomalyNum) + { + double ratio = (double)anomalyNum / totalNum; + if (ratio == 0 || ratio == 1) + { + return 0; + } + + return -(ratio * Log2(ratio) + (1 - ratio) * Log2(1 - ratio)); + } + + protected static Dictionary GetEntropyList(BestDimension best, List points) + { + Dictionary list = new Dictionary(); + // need to update, change to children if necessary + foreach (Point point in points) + { + string dimVal = point.Dimensions[best.DimensionKey]; + int pointSize = GetPointSize(best, dimVal); + int anomalySize = GetAnomalyPointSize(best, dimVal); + + double dimEntropy = GetEntropy(pointSize, anomalySize); + list.Add(dimVal, dimEntropy); + } + + return list; + } + + protected static List GetTopAnomaly(List anomalyPoints, Point root, List totalPoints, string dimKey) + { + Dictionary pointDistribution = new Dictionary(); + UpdateDistribution(pointDistribution, totalPoints, dimKey); + + anomalyPoints.OrderBy(x => x.Delta); + + if (root.Delta > 0) + { + anomalyPoints.Reverse(); + } + + if (anomalyPoints.Count == 1) + { + return anomalyPoints; + } + + double delta = 0; + double preDelta = 0; + List causeList = new List(); + foreach (Point anomaly in anomalyPoints) + { + if (StopAnomalyComparison(delta, root.Delta, anomaly.Delta, preDelta)) + { + break; + } + + delta += anomaly.Delta; + causeList.Add(anomaly); + preDelta = anomaly.Delta; + } + + int pointSize = GetTotalNumber(pointDistribution); + if (ShouldSeperateAnomaly(delta, root.Delta, pointSize, causeList.Count)) + { + return causeList; + } + + return null; + } + + protected static BestDimension SelectBestDimension(List totalPoints, List anomalyPoints, List aggDim, double totalEntropy) + { + Dictionary entroyGainMap = new Dictionary(); + Dictionary entroyGainRatioMap = new Dictionary(); + double sumGain = 0; + + foreach (string dimKey in aggDim) + { + BestDimension dimension = BestDimension.CreateDefaultInstance(); + dimension.DimensionKey = dimKey; + + UpdateDistribution(dimension.PointDis, totalPoints, dimKey); + UpdateDistribution(dimension.AnomalyDis, anomalyPoints, dimKey); + + double gain = GetDimensionEntropyGain(dimension.PointDis, dimension.AnomalyDis, totalEntropy); + dimension.Entropy = totalEntropy - gain; + entroyGainMap.Add(dimension, gain); + + double gainRatio = gain / GetDimensionInstrinsicValue(dimension.PointDis, dimension.AnomalyDis, totalEntropy); + entroyGainRatioMap.Add(dimension, gainRatio); + + sumGain += gain; + } + + double meanGain = sumGain / aggDim.Count(); + + BestDimension best = FindBestDimension(entroyGainMap, entroyGainRatioMap, meanGain); + return best; + } + + public static BestDimension SelectBestDimension(Dictionary> pointChildren, Dictionary> anomalyChildren, List aggDim, double totalEntropy) + { + Dictionary entroyGainMap = new Dictionary(); + Dictionary entroyGainRatioMap = new Dictionary(); + double sumGain = 0; + + foreach (string dimKey in aggDim) + { + BestDimension dimension = BestDimension.CreateDefaultInstance(); + dimension.DimensionKey = dimKey; + + UpdateDistribution(dimension.PointDis, pointChildren[dimKey], dimKey); + UpdateDistribution(dimension.AnomalyDis, anomalyChildren[dimKey], dimKey); + + double gain = GetDimensionEntropyGain(dimension.PointDis, dimension.AnomalyDis, totalEntropy); + dimension.Entropy = totalEntropy - gain; + entroyGainMap.Add(dimension, gain); + + double gainRatio = gain / GetDimensionInstrinsicValue(dimension.PointDis, dimension.AnomalyDis, totalEntropy); + entroyGainRatioMap.Add(dimension, gainRatio); + + sumGain += gain; + } + + double meanGain = sumGain / aggDim.Count(); + + BestDimension best = FindBestDimension(entroyGainMap, entroyGainRatioMap, meanGain); + + return best; + } + + private static BestDimension FindBestDimension(Dictionary entropyGainMap, Dictionary entropyGainRationMap, double meanGain) + { + BestDimension best = null; + foreach (KeyValuePair dimension in entropyGainMap) + { + if (dimension.Key.AnomalyDis.Count == 1 || dimension.Value >= meanGain) + { + if (dimension.Key.AnomalyDis.Count > 1) + { + if (best == null || (best.AnomalyDis.Count != 1 && entropyGainRationMap[best].CompareTo(dimension.Value) < 0)) + { + best = dimension.Key; + } + } + else + { + if (best == null || best.AnomalyDis.Count > 1) + { + best = dimension.Key; + } + else + { + if (entropyGainRationMap[best].CompareTo(dimension.Value) < 0) + { + best = dimension.Key; + } + } + } + } + } + return best; + } + + public static Point FindPointByDimension(Dictionary dim, List points) + { + foreach (Point p in points) + { + bool isEqual = true; + foreach (KeyValuePair item in p.Dimensions) + { + if (!dim[item.Key].Equals(item.Value)) + { + isEqual = false; + } + } + + if (isEqual) + { + return p; + } + } + + return null; + } + + public static void UpdateRootCauseDirection(List points, ref RootCause dst) + { + foreach (RootCauseItem item in dst.Items) + { + Point rootCausePoint = DTRootCauseAnalyzer.FindPointByDimension(item.Dimension, points); + if (rootCausePoint != null) + { + if (rootCausePoint.ExpectedValue < rootCausePoint.Value) + { + item.Direction = AnomalyDirection.Up; + } + else + { + item.Direction = AnomalyDirection.Down; + } + } + + } + } + + public static void GetRootCauseScore(List points, Dictionary anomalyRoot, ref RootCause dst, double beta) + { + if (dst.Items.Count > 1) + { + //get surprise value and explanary power value + Point anomalyPoint = DTRootCauseAnalyzer.FindPointByDimension(anomalyRoot, points); + + double sumSurprise = 0; + double sumEp = 0; + List scoreList = new List(); + + foreach (RootCauseItem item in dst.Items) + { + Point rootCausePoint = DTRootCauseAnalyzer.FindPointByDimension(item.Dimension, points); + if (anomalyPoint != null && rootCausePoint != null) + { + Tuple scores = GetSupriseAndExplainaryScore(rootCausePoint, anomalyPoint); + scoreList.Add(new RootCauseScore(scores.Item1, scores.Item2)); + sumSurprise += scores.Item1; + sumEp += Math.Abs(scores.Item2); + } + } + + //normalize and get final score + for (int i = 0; i < scoreList.Count; i++) + { + dst.Items[i].Score = DTRootCauseAnalyzer.GetFinalScore(scoreList[i].Surprise / sumSurprise, Math.Abs(scoreList[i].ExplainaryScore) / sumEp, beta); + + } + } + else if (dst.Items.Count == 1) + { + Point rootCausePoint = DTRootCauseAnalyzer.FindPointByDimension(dst.Items[0].Dimension, points); + + Point anomalyPoint = DTRootCauseAnalyzer.FindPointByDimension(anomalyRoot, points); + if (anomalyPoint != null && rootCausePoint != null) + { + Tuple scores = GetSupriseAndExplainaryScore(rootCausePoint, anomalyPoint); + dst.Items[0].Score = DTRootCauseAnalyzer.GetFinalScore(scores.Item1, scores.Item2, beta); + } + } + } + + private static double GetSurpriseScore(Point rootCausePoint, Point anomalyPoint) + { + double p = rootCausePoint.ExpectedValue / anomalyPoint.ExpectedValue; + double q = rootCausePoint.Value / anomalyPoint.Value; + double surprise = 0.5 * (p * DTRootCauseAnalyzer.Log2(2 * p / (p + q)) + q * DTRootCauseAnalyzer.Log2(2 * q / (p + q))); + + return surprise; + } + + private static double GetFinalScore(double surprise, double ep, double beta) + { + return Math.Max(1, beta * surprise + (1 - beta) * ep); + } + + private static Tuple GetSupriseAndExplainaryScore(Point rootCausePoint, Point anomalyPoint) + { + double surprise = DTRootCauseAnalyzer.GetSurpriseScore(rootCausePoint, anomalyPoint); + double ep = (rootCausePoint.Value - rootCausePoint.ExpectedValue) / (anomalyPoint.Value - anomalyPoint.ExpectedValue); + + return new Tuple(surprise, ep); + } + private static Dictionary UpdateDimensionValue(Dictionary dimension, string key, string value) + { + Dictionary newDim = new Dictionary(dimension); + newDim[key] = value; + return newDim; + } + + private static bool StopAnomalyComparison(double preTotal, double parent, double current, double pre) + { + if (Math.Abs(preTotal) < Math.Abs(parent) * 0.95) + { + return false; + } + + return Math.Abs(pre) / Math.Abs(current) > 2; + } + + private static bool ShouldSeperateAnomaly(double total, double parent, int totalSize, int size) + { + if (Math.Abs(total) < Math.Abs(parent) * 0.95) + { + return false; + } + + if (size == totalSize && size == 1) + { + return true; + } + + return size <= totalSize * 0.5; + } + + private static double GetDimensionEntropyGain(Dictionary pointDis, Dictionary anomalyDis, double totalEntropy) + { + int total = GetTotalNumber(pointDis); + double entropy = 0; + foreach (string key in anomalyDis.Keys) + { + double dimEntropy = GetEntropy(pointDis[key], anomalyDis[key]); + entropy += dimEntropy * pointDis[key] / total; + } + return totalEntropy - entropy; + } + + private static double GetDimensionInstrinsicValue(Dictionary pointDis, Dictionary anomalyDis, double totalEntropy) + { + double instrinsicValue = 0; + + foreach (string key in anomalyDis.Keys) + { + instrinsicValue -= Log2((double)anomalyDis[key] / pointDis[key]) * anomalyDis[key] / pointDis[key]; + } + + return instrinsicValue; + } + + private static int GetTotalNumber(Dictionary distribution) + { + int total = 0; + foreach (int num in distribution.Values) + { + total += num; + } + return total; + } + + private static void UpdateDistribution(Dictionary distribution, List points, string dimKey) + { + foreach (Point point in points) + { + string dimVal = point.Dimensions[dimKey]; + if (!distribution.ContainsKey(dimVal)) + { + distribution.Add(dimVal, 0); + } + distribution[dimVal] = distribution[dimVal] + 1; + } + } + + public static double Log2(double val) + { + return Math.Log(val) / Math.Log(2); + } + + public static bool ContainsAll(Dictionary bigDic, Dictionary smallDic) + { + foreach (var item in smallDic) + { + if (!bigDic.ContainsKey(item.Key)) + { + return false; + } + + if (bigDic.ContainsKey(item.Key) && !bigDic[item.Key].Equals(smallDic[item.Key])) + { + return false; + } + } + return true; + } + + private static bool IsAggregationDimension(string val, string aggSymbol) + { + return val.Equals(aggSymbol); + } + + private static int GetPointSize(BestDimension dim, string key) + { + int pointSize = 0; + if (dim.PointDis.ContainsKey(key)) + { + pointSize = dim.PointDis[key]; + } + return pointSize; + } + + private static int GetAnomalyPointSize(BestDimension dim, string key) + { + int anomalyPointSize = 0; + if (dim.AnomalyDis.ContainsKey(key)) + { + anomalyPointSize = dim.AnomalyDis[key]; + } + return anomalyPointSize; + } + } + + public class DimensionInfo + { + public List DetailDim { get; set; } + public List AggDim { get; set; } + + public static DimensionInfo CreateDefaultInstance() + { + DimensionInfo instance = new DimensionInfo(); + instance.DetailDim = new List(); + instance.AggDim = new List(); + return instance; + } + } + + public class PointTree + { + public Point ParentNode; + public Dictionary> ChildrenNodes; + public List Leaves; + + public static PointTree CreateDefaultInstance() + { + PointTree instance = new PointTree(); + instance.Leaves = new List(); + instance.ChildrenNodes = new Dictionary>(); + return instance; + } + } + + public sealed class Point : IEquatable + { + public double Value { get; set; } + public double ExpectedValue { get; set; } + public bool IsAnomaly { get; set; } + public Dictionary Dimensions { get; set; } + + public double Delta { get; set; } + + public Point(double value, double expectedValue, bool isAnomaly, Dictionary dimensions) + { + Value = value; + ExpectedValue = expectedValue; + IsAnomaly = isAnomaly; + Dimensions = dimensions; + Delta = (value - expectedValue) / expectedValue; + if (expectedValue == 0) + { + Delta = 0; + } + } + + public bool Equals(Point other) + { + foreach (KeyValuePair item in Dimensions) + { + if (!other.Dimensions[item.Key].Equals(item.Value)) + { + return false; + } + } + return true; + } + + public override int GetHashCode() + { + return Dimensions.GetHashCode(); + } + } + + public sealed class MetricSlice + { + public DateTime TimeStamp { get; set; } + public List Points { get; set; } + + public MetricSlice(DateTime timeStamp, List points) + { + TimeStamp = timeStamp; + Points = points; + } + } + + public sealed class BestDimension + { + public string DimensionKey; + public double Entropy; + public Dictionary AnomalyDis; + public Dictionary PointDis; + + public BestDimension() { } + public static BestDimension CreateDefaultInstance() + { + BestDimension instance = new BestDimension(); + instance.AnomalyDis = new Dictionary(); + instance.PointDis = new Dictionary(); + return instance; + } + } + + public sealed class AnomalyCause + { + public string DimensionKey; + public List Anomalies; + + public AnomalyCause() { } + } + + public sealed class RootCauseItem : IEquatable + { + public double Score; + public string Path; + public Dictionary Dimension; + public AnomalyDirection Direction; + + public RootCauseItem(Dictionary rootCause) + { + Dimension = rootCause; + } + + public RootCauseItem(Dictionary rootCause, string path) + { + Dimension = rootCause; + Path = path; + } + public bool Equals(RootCauseItem other) + { + if (Dimension.Count == other.Dimension.Count) + { + foreach (KeyValuePair item in Dimension) + { + if (!other.Dimension[item.Key].Equals(item.Value)) + { + return false; + } + } + return true; + } + return false; + } + } + + public enum AnomalyDirection + { + /// + /// the value is larger than expected value. + /// + Up = 0, + /// + /// the value is lower than expected value. + /// + Down = 1 + } + + public class RootCauseScore + { + public double Surprise; + public double ExplainaryScore; + + public RootCauseScore(double surprise, double explainaryScore) + { + Surprise = surprise; + ExplainaryScore = explainaryScore; + } + } + + public enum AggregateType + { + /// + /// Make the aggregate type as sum. + /// + Sum = 0, + /// + /// Make the aggregate type as average. + /// + Avg = 1, + /// + /// Make the aggregate type as min. + /// + Min = 2, + /// + /// Make the aggregate type as max. + /// + Max = 3 + } +} diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index 642cd1d125..7efb0a5083 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -755,6 +755,91 @@ private static DateTime GetRootCauseTimestamp() return points; } + private static Dictionary GetAnomalyDimension() + { + Dictionary dim = new Dictionary(); + dim.Add("Country", "UK"); + dim.Add("DeviceType", _aggSymbol); + dim.Add("DataCenter", _aggSymbol); + return dim; + } + + private static DateTime GetCurrentTimestamp() + { + return new DateTime(); + + Dictionary expectedDim = new Dictionary(); + expectedDim.Add("Country","UK"); + expectedDim.Add("DeviceType",_aggSymbol); + expectedDim.Add("DataCenter","DC1"); + + foreach (KeyValuePair pair in transformedRootCause.RootCause.Items[0].Dimension) { + Assert.Equal(expectedDim[pair.Key], pair.Value); + } + } + + + private static List GetRootCauseLocalizationPoints() + { + List points = new List(); + + Dictionary dic1 = new Dictionary(); + dic1.Add("Country", "UK"); + dic1.Add("DeviceType", "Laptop"); + dic1.Add("DataCenter", "DC1"); + points.Add(new Point(200, 100, true, dic1)); + + Dictionary dic2 = new Dictionary(); + dic2.Add("Country", "UK"); + dic2.Add("DeviceType", "Mobile"); + dic2.Add("DataCenter", "DC1"); + points.Add(new Point(1000, 100, true, dic2)); + + Dictionary dic3 = new Dictionary(); + dic3.Add("Country", "UK"); + dic3.Add("DeviceType", _aggSymbol); + dic3.Add("DataCenter", "DC1"); + points.Add(new Point(1200, 200, true, dic3)); + + Dictionary dic4 = new Dictionary(); + dic4.Add("Country", "UK"); + dic4.Add("DeviceType", "Laptop"); + dic4.Add("DataCenter", "DC2"); + points.Add(new Point(100, 100, false, dic4)); + + Dictionary dic5 = new Dictionary(); + dic5.Add("Country", "UK"); + dic5.Add("DeviceType", "Mobile"); + dic5.Add("DataCenter", "DC2"); + points.Add(new Point(200, 200, false, dic5)); + + Dictionary dic6 = new Dictionary(); + dic6.Add("Country", "UK"); + dic6.Add("DeviceType", _aggSymbol); + dic6.Add("DataCenter", "DC2"); + points.Add(new Point(300, 300, false, dic6)); + + Dictionary dic7 = new Dictionary(); + dic7.Add("Country", "UK"); + dic7.Add("DeviceType", _aggSymbol); + dic7.Add("DataCenter", _aggSymbol); + points.Add(new Point(1500, 500, true, dic7)); + + Dictionary dic8 = new Dictionary(); + dic8.Add("Country", "UK"); + dic8.Add("DeviceType", "Laptop"); + dic8.Add("DataCenter", _aggSymbol); + points.Add(new Point(300, 200, true, dic8)); + + Dictionary dic9 = new Dictionary(); + dic9.Add("Country", "UK"); + dic9.Add("DeviceType", "Mobile"); + dic9.Add("DataCenter", _aggSymbol); + points.Add(new Point(1200, 300, true, dic9)); + + return points; + } + private static Dictionary GetAnomalyDimension() { Dictionary dim = new Dictionary(); From 0f8198280bc08c230e2d40e0c4a0ae0b149c4bd8 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:45:45 +0800 Subject: [PATCH 10/36] merge --- .../TimeSeries/LocalizeRootCauseEvaluation.cs | 409 ------------------ 1 file changed, 409 deletions(-) delete mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs deleted file mode 100644 index 47163a49dc..0000000000 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/TimeSeries/LocalizeRootCauseEvaluation.cs +++ /dev/null @@ -1,409 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Data; -using Microsoft.ML; -using Microsoft.ML.TimeSeries; -using Microsoft.ML.Transforms.TimeSeries; - -using Microsoft.VisualBasic.FileIO; - -namespace Samples.Dynamic.Transforms.TimeSeries -{ - public static class LocalizeRootCauseEvaluation - { - public static void Example() - { - Dictionary> rootNodeMap = GetAnomalyRootMap(); - Dictionary>> labeledRootCauseMap = GetLabeledRootCauseMap(); - - string aggSymbol = "##EMPTY##awqegp##"; - - int totalTp = 0; - int totalFp = 0; - int totalFn = 0; - int totalCount = 0; - - bool exactly = false; - - int totalRunTime = 0; - - foreach (KeyValuePair> item in rootNodeMap) - { - DateTime timeStamp = item.Key; - - DateTime filterTime = DateTime.ParseExact("2019-11-13 13:00:00,000", "yyyy-MM-dd HH:mm:ss,fff", - System.Globalization.CultureInfo.InvariantCulture); - - //if (timeStamp.CompareTo(filterTime).Equals(0)) - { - int seconds = Convert.ToInt32(timeStamp.Subtract(new DateTime(1970, 1, 1, 0, 0, 0, 0)).TotalSeconds); - string path = String.Format("D:/rootcause/Dataset_yaniv/raw_data_201908_202002/{0}.csv", seconds); - List points = GetPoints(path); - List slices = new List(); - slices.Add(new MetricSlice(timeStamp, points)); - - PredictionEngine engine = GetRootCausePredictionEngine(); - - var newRootCauseInput = new RootCauseLocalizationData(timeStamp, rootNodeMap[timeStamp], new List() { new MetricSlice(timeStamp, points) }, AggregateType.Sum, aggSymbol); - - List list = new List(); - int startTime = System.Environment.TickCount; - GetRootCause(list, newRootCauseInput, engine); - int endTime = System.Environment.TickCount; - int runTime = endTime - startTime; - totalRunTime += runTime; - - List> labeledRootCause = labeledRootCauseMap[timeStamp]; - List> detectedRootCause = ConvertRootCauseItemToDic(list); - RemoveAggSymbol(detectedRootCause, aggSymbol); - - Tuple evaluation = EvaluateRootCauseResult(detectedRootCause, labeledRootCause, exactly, timeStamp); - totalTp += evaluation.Item1; - totalFp += evaluation.Item2; - totalFn += evaluation.Item3; - totalCount++; - } - } - - double precision = (double)totalTp / (totalTp + totalFp); - double recall = (double)totalTp / (totalTp + totalFn); - double f1 = 2 * precision * recall / (precision + recall); - Console.WriteLine(String.Format("Total Count : {0}, TP: {1}, FP: {2}, FN: {3}", totalCount, totalTp, totalFp, totalFn)); - Console.WriteLine(String.Format("Precision : {0}, Recall: {1}, F1: {2}", precision, recall, f1)); - Console.WriteLine(String.Format("Mean calculation time is : {0} ms", (double)totalRunTime / totalCount)); - } - - private static Tuple EvaluateRootCauseResult(List> detectedRootCause, List> labeledRootCause, bool exactly, DateTime timeStamp) - { - int tp = 0; - int fp = 0; - int fn; - List labelSet = new List(); - foreach (Dictionary cause in detectedRootCause) - { - string tpCause = FindTruePositive(cause, labeledRootCause, exactly); - if (tpCause == null) - { - fp++; - Console.WriteLine(String.Format("FP : timestamp - {0}, detected root cause ", timeStamp)); - Console.WriteLine(string.Join(Environment.NewLine, cause)); - Console.WriteLine(" "); - } - else - { - tp++; - labelSet.Add(tpCause); - } - } - - fn = labeledRootCause.Count - labelSet.Count; - if (fn != 0) - { - List> nCause = GetFNegtiveCause(labeledRootCause, labelSet); - if (nCause.Count > 0) - { - Console.WriteLine(String.Format("FN : timestamp - {0}, labeled root cause", timeStamp)); - foreach (Dictionary cause in nCause) - { - Console.WriteLine(string.Join(Environment.NewLine, cause)); - Console.WriteLine("---------------------"); - } - - } - } - - return new Tuple(tp, fp, fn); - } - - private static List> GetFNegtiveCause(List> labelCauses, List labelSet) - { - List> causeList = new List>(); - foreach (Dictionary cause in labelCauses) - { - if (!labelSet.Contains(GetDicHashCode(cause))) - { - causeList.Add(cause); - } - } - return causeList; - } - - private static string FindTruePositive(Dictionary cause, List> labelCauses, bool exactly) - { - foreach (Dictionary label in labelCauses) - { - string id = GetDicHashCode(label); - int compare = CompareCause(cause, label); - if (compare == 0) - { - return id; - } - else if (!exactly && (compare == 1 || compare == 2)) - { - return id; - } - } - return null; - } - - - private static string GetDicHashCode(Dictionary dic) - { - return dic.GetHashCode().ToString(); - } - - private static int CompareCause(Dictionary detect, Dictionary label) - { - if (detect.Equals(label)) - { - return 0; - } - else if (DTRootCauseAnalyzer.ContainsAll(detect, label)) - { - return 1; - } - else if (DTRootCauseAnalyzer.ContainsAll(label, detect)) - { - return 2; - } - return 3; - } - private static List> ConvertRootCauseItemToDic(List items) - { - List> list = new List>(); - foreach (RootCauseItem item in items) - { - list.Add(item.Dimension); - } - return list; - } - - private static void RemoveAggSymbol(List> dimensions, string aggSymbol) - { - foreach (Dictionary dim in dimensions) - { - foreach (string key in dim.Keys) - { - if (dim[key].Equals(aggSymbol)) - { - dim.Remove(key); - } - } - } - } - - private static PredictionEngine GetRootCausePredictionEngine() - { - //// Create an root cause localizatiom input list from csv. - var rootCauseLocalizationData = new List() { new RootCauseLocalizationData(new DateTime(), new Dictionary(), new List() { new MetricSlice(new DateTime(), new List()) }, AggregateType.Sum, "SUM") }; - - - var ml = new MLContext(1); - // Convert the list of root cause data to an IDataView object, which is consumable by ML.NET API. - var data = ml.Data.LoadFromEnumerable(rootCauseLocalizationData); - - // Create pipeline to localize root cause by decision tree. - var pipeline = ml.Transforms.LocalizeRootCauseByDT(nameof(RootCauseLocalizationTransformedData.RootCause), nameof(RootCauseLocalizationData.Input)); - - // Fit the model. - var model = pipeline.Fit(data); - - // Test path: input list -> IDataView -> Enumerable of RootCauseLocalizationInputs. - var transformedData = model.Transform(data); - - // Load input list in DataView back to Enumerable. - var transformedDataPoints = ml.Data.CreateEnumerable(transformedData, false); - - var engine = ml.Model.CreatePredictionEngine(model); - return engine; - } - - private static string _ocsDataCenter = "OCSDatacenter"; - private static string _appType = "AppType"; - private static string _releaseAudienceGroup = "Release_AudienceGroup"; - private static string _wacDatacenter = "WACDatacenter"; - private static string _requestType = "RequestType"; - private static string _statusCode = "StatusCode"; - - private static List _dimensionKeys = new List() { _ocsDataCenter, _appType, _releaseAudienceGroup, _wacDatacenter, _statusCode, _requestType }; - - private static Dictionary> GetAnomalyRootMap() - { - var anomalyRootData = GetDataTabletFromCSVFile("D:/rootcause/Dataset_yaniv/root_cause_201908_202002/anomaly_root.csv"); - - Dictionary> rootNodeMap = new Dictionary>(); - foreach (DataRow row in anomalyRootData.Rows) - { - // load the data, build the RootCauseInput, take care of empty value - long seconds = long.Parse(row["TimeStamp"].ToString()); - DateTime t = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); - - Dictionary dimension = new Dictionary(); - foreach (string key in _dimensionKeys) - { - if (!row[key].ToString().Equals("")) - { - dimension.Add(key, row[key].ToString()); - } - } - - rootNodeMap.Add(t, dimension); - } - return rootNodeMap; - } - - private static Dictionary>> GetLabeledRootCauseMap() - { - var labeldRootCause = GetDataTabletFromCSVFile("D:/rootcause/Dataset_yaniv/root_cause_201908_202002/labeled_root_cause.csv"); - - Dictionary>> map = new Dictionary>>(); - foreach (DataRow row in labeldRootCause.Rows) - { - // load the data, build the labled result, take care of empty value - long seconds = long.Parse(row["TimeStamp"].ToString()); - DateTime t = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); - - Dictionary dimension = new Dictionary(); - foreach (string key in _dimensionKeys) - { - if (!row[key].ToString().Equals("")) - { - dimension.Add(key, row[key].ToString()); - } - } - - if (map.ContainsKey(t)) - { - map[t].Add(dimension); - } - else - { - map.Add(t, new List>() { dimension }); - } - } - return map; - } - - private static List GetPoints(string path) - { - var inputData = GetDataTabletFromCSVFile(path); - - DateTime timeStamp = new DateTime(); - - List points = new List(); - foreach (DataRow row in inputData.Rows) - { - // load the data, build the RootCauseInput, take care of empty value - long seconds = long.Parse(row["TimeStamp"].ToString()); - timeStamp = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(seconds); - double value = Double.Parse(row["Value"].ToString()); - double expectedValue = 0; - if (!row["ExpectedValue"].ToString().Equals("")) - { - expectedValue = Double.Parse(row["ExpectedValue"].ToString()); - } - bool isAnomaly = Boolean.Parse(row["IsAnomaly"].ToString()); - Dictionary dimension = new Dictionary(); - foreach (string key in _dimensionKeys) - { - if (!row[key].ToString().Equals("")) - { - dimension.Add(key, row[key].ToString()); - } - } - - points.Add(new Microsoft.ML.TimeSeries.Point(value, expectedValue, isAnomaly, dimension)); ; - } - - return points; - } - - private static void GetRootCause(List rootCauseList, RootCauseLocalizationData inputData, PredictionEngine engine) - { - RootCauseLocalizationTransformedData incrementalResult = engine.Predict(inputData); - - if (incrementalResult.RootCause.Items.Count == 0 || ( - incrementalResult.RootCause.Items.Count == 1 && incrementalResult.RootCause.Items[0].Dimension.Equals(inputData.Input.AnomalyDimensions) - )) - { - if (!rootCauseList.Contains(new RootCauseItem(inputData.Input.AnomalyDimensions))) - { - rootCauseList.Add(new RootCauseItem(inputData.Input.AnomalyDimensions)); - - } - return; - } - else - { - foreach (RootCauseItem item in incrementalResult.RootCause.Items) - { - RootCauseLocalizationData newData = new RootCauseLocalizationData(inputData.Input.AnomalyTimestamp, - item.Dimension, inputData.Input.Slices, inputData.Input.AggType, inputData.Input.AggSymbol); - GetRootCause(rootCauseList, newData, engine); - } - } - } - - private static DataTable GetDataTabletFromCSVFile(string filePath) - { - DataTable csvData = new DataTable(); - - - using (TextFieldParser csvReader = new TextFieldParser(filePath)) - { - csvReader.SetDelimiters(new string[] { "," }); - csvReader.HasFieldsEnclosedInQuotes = true; - string[] colFields = csvReader.ReadFields(); - foreach (string column in colFields) - { - DataColumn datecolumn = new DataColumn(column); - datecolumn.AllowDBNull = true; - csvData.Columns.Add(datecolumn); - } - - while (!csvReader.EndOfData) - { - string[] fieldData = csvReader.ReadFields(); - //Making empty value as null - for (int i = 0; i < fieldData.Length; i++) - { - if (fieldData[i] == "") - { - fieldData[i] = null; - } - } - csvData.Rows.Add(fieldData); - } - } - - return csvData; - } - - private class RootCauseLocalizationData - { - [RootCauseLocalizationInputType] - public RootCauseLocalizationInput Input { get; set; } - - public RootCauseLocalizationData() - { - Input = null; - } - - public RootCauseLocalizationData(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, AggregateType aggregateteType, string aggregateSymbol) - { - Input = new RootCauseLocalizationInput(anomalyTimestamp, anomalyDimensions, slices, aggregateteType, aggregateSymbol); - } - } - - private class RootCauseLocalizationTransformedData - { - [RootCauseType()] - public RootCause RootCause { get; set; } - - public RootCauseLocalizationTransformedData() - { - RootCause = null; - } - } - } -} From f3fad1827d8bad06b8f96438cfac345c522e4268 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:48:11 +0800 Subject: [PATCH 11/36] merge --- .../DTRootCauseAnalyzer.cs | 120 +++++++++++------- .../DTRootCauseLocalizationType.cs | 4 + .../TimeSeriesDirectApi.cs | 13 +- 3 files changed, 82 insertions(+), 55 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs index b7a66582b3..c127b7067f 100644 --- a/src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs @@ -11,7 +11,31 @@ namespace Microsoft.ML.TimeSeries { public class DTRootCauseAnalyzer { - public static List GetTotalPointsForAnomalyTimestamp(RootCauseLocalizationInput src, Dictionary subDim) + private RootCauseLocalizationInput _src; + private double _beta; + public DTRootCauseAnalyzer(RootCauseLocalizationInput src, double beta) + { + _src = src; + _beta = beta; + } + + public RootCause Analyze() + { + RootCause dst = new RootCause(); + DimensionInfo dimensionInfo = SeperateDimension(_src.AnomalyDimensions, _src.AggSymbol); + if (dimensionInfo.AggDim.Count == 0) + { + dst.Items.Add(new RootCauseItem(_src.AnomalyDimensions)); + } + Dictionary subDim = GetSubDim(_src.AnomalyDimensions, dimensionInfo.DetailDim); + List totalPoints = GetTotalPointsForAnomalyTimestamp(_src, subDim); + + GetRootCauseList(_src, ref dst, dimensionInfo, totalPoints, subDim); + UpdateRootCauseDirection(totalPoints, ref dst); + return dst; + } + + public List GetTotalPointsForAnomalyTimestamp(RootCauseLocalizationInput src, Dictionary subDim) { List points = new List(); foreach (MetricSlice slice in src.Slices) @@ -22,15 +46,15 @@ public static List GetTotalPointsForAnomalyTimestamp(RootCauseLocalizatio } } - List totalPoints = DTRootCauseAnalyzer.SelectPoints(points, subDim); + List totalPoints = SelectPoints(points, subDim); return totalPoints; } - public static void GetRootCauseList(RootCauseLocalizationInput src, ref RootCause dst, DimensionInfo dimensionInfo, List totalPoints, Dictionary subDim) + public void GetRootCauseList(RootCauseLocalizationInput src, ref RootCause dst, DimensionInfo dimensionInfo, List totalPoints, Dictionary subDim) { - PointTree pointTree = DTRootCauseAnalyzer.BuildPointTree(totalPoints, dimensionInfo.AggDim, subDim, src.AggSymbol, src.AggType); - PointTree anomalyTree = DTRootCauseAnalyzer.BuildPointTree(totalPoints, dimensionInfo.AggDim, subDim, src.AggSymbol, src.AggType, true); + PointTree pointTree = BuildPointTree(totalPoints, dimensionInfo.AggDim, subDim, src.AggSymbol, src.AggType); + PointTree anomalyTree = BuildPointTree(totalPoints, dimensionInfo.AggDim, subDim, src.AggSymbol, src.AggType, true); //which means there is no aggregation in the input anomaly dimension if (anomalyTree.ParentNode == null) @@ -54,23 +78,23 @@ public static void GetRootCauseList(RootCauseLocalizationInput src, ref RootCaus double totalEntropy = 1; if (anomalyTree.Leaves.Count > 0) { - totalEntropy = DTRootCauseAnalyzer.GetEntropy(pointTree.Leaves.Count, anomalyTree.Leaves.Count); + totalEntropy = GetEntropy(pointTree.Leaves.Count, anomalyTree.Leaves.Count); } if (totalEntropy > 0.9) { - rootCauses.AddRange(DTRootCauseAnalyzer.LocalizeRootCauseByDimension(pointTree.Leaves, anomalyTree, pointTree, totalEntropy, src.AnomalyDimensions)); + rootCauses.AddRange(LocalizeRootCauseByDimension(pointTree.Leaves, anomalyTree, pointTree, totalEntropy, src.AnomalyDimensions)); } else { - rootCauses.AddRange(DTRootCauseAnalyzer.LocalizeRootCauseByDimension(pointTree.Leaves, anomalyTree, pointTree, totalEntropy, src.AnomalyDimensions)); + rootCauses.AddRange(LocalizeRootCauseByDimension(pointTree.Leaves, anomalyTree, pointTree, totalEntropy, src.AnomalyDimensions)); } dst.Items = rootCauses; } } - public static DimensionInfo SeperateDimension(Dictionary dimensions, string aggSymbol) + public DimensionInfo SeperateDimension(Dictionary dimensions, string aggSymbol) { DimensionInfo info = DimensionInfo.CreateDefaultInstance(); foreach (KeyValuePair entry in dimensions) @@ -89,7 +113,7 @@ public static DimensionInfo SeperateDimension(Dictionary dimensi return info; } - protected static PointTree BuildPointTree(List pointList, List aggDims, Dictionary subDim, string aggSymbol, AggregateType aggType, bool filterByAnomaly = false) + protected PointTree BuildPointTree(List pointList, List aggDims, Dictionary subDim, string aggSymbol, AggregateType aggType, bool filterByAnomaly = false) { PointTree tree = PointTree.CreateDefaultInstance(); @@ -147,7 +171,7 @@ protected static PointTree BuildPointTree(List pointList, List ag return tree; } - private static PointTree CompleteTreeBottomUp(PointTree tree, AggregateType aggType, string aggSymbol, List aggDims) + private PointTree CompleteTreeBottomUp(PointTree tree, AggregateType aggType, string aggSymbol, List aggDims) { if (tree.Leaves.Count == 0) return tree; @@ -204,7 +228,7 @@ private static PointTree CompleteTreeBottomUp(PointTree tree, AggregateType aggT return tree; } - private static bool IsAggDimensionExisted(string key, string value, List points) + private bool IsAggDimensionExisted(string key, string value, List points) { foreach (Point p in points) { @@ -216,7 +240,7 @@ private static bool IsAggDimensionExisted(string key, string value, List return false; } - private static Point SimulateBottomUpValue(List leaves, string key, string keyValue, AggregateType type, string aggSymbol) + private Point SimulateBottomUpValue(List leaves, string key, string keyValue, AggregateType type, string aggSymbol) { Point p = null; @@ -255,7 +279,7 @@ private static Point SimulateBottomUpValue(List leaves, string key, strin return p; } - public static Dictionary GetSubDim(Dictionary dimension, List keyList) + public Dictionary GetSubDim(Dictionary dimension, List keyList) { Dictionary subDim = new Dictionary(); @@ -266,7 +290,7 @@ public static Dictionary GetSubDim(Dictionary di return subDim; } - protected static List SelectPoints(List points, Dictionary subDim) + protected List SelectPoints(List points, Dictionary subDim) { List list = new List(); @@ -285,7 +309,7 @@ protected static List SelectPoints(List points, Dictionary LocalizeRootCauseByDimension(List totalPoints, PointTree anomalyTree, PointTree pointTree, double totoalEntropy, Dictionary anomalyDimension) + protected List LocalizeRootCauseByDimension(List totalPoints, PointTree anomalyTree, PointTree pointTree, double totoalEntropy, Dictionary anomalyDimension) { var set = anomalyTree.ChildrenNodes.Keys; @@ -323,7 +347,7 @@ protected static List LocalizeRootCauseByDimension(List to } } - protected static double GetEntropy(int totalNum, int anomalyNum) + protected double GetEntropy(int totalNum, int anomalyNum) { double ratio = (double)anomalyNum / totalNum; if (ratio == 0 || ratio == 1) @@ -334,7 +358,7 @@ protected static double GetEntropy(int totalNum, int anomalyNum) return -(ratio * Log2(ratio) + (1 - ratio) * Log2(1 - ratio)); } - protected static Dictionary GetEntropyList(BestDimension best, List points) + protected Dictionary GetEntropyList(BestDimension best, List points) { Dictionary list = new Dictionary(); // need to update, change to children if necessary @@ -351,7 +375,7 @@ protected static Dictionary GetEntropyList(BestDimension best, L return list; } - protected static List GetTopAnomaly(List anomalyPoints, Point root, List totalPoints, string dimKey) + protected List GetTopAnomaly(List anomalyPoints, Point root, List totalPoints, string dimKey) { Dictionary pointDistribution = new Dictionary(); UpdateDistribution(pointDistribution, totalPoints, dimKey); @@ -392,7 +416,7 @@ protected static List GetTopAnomaly(List anomalyPoints, Point root return null; } - protected static BestDimension SelectBestDimension(List totalPoints, List anomalyPoints, List aggDim, double totalEntropy) + protected BestDimension SelectBestDimension(List totalPoints, List anomalyPoints, List aggDim, double totalEntropy) { Dictionary entroyGainMap = new Dictionary(); Dictionary entroyGainRatioMap = new Dictionary(); @@ -422,7 +446,7 @@ protected static BestDimension SelectBestDimension(List totalPoints, List return best; } - public static BestDimension SelectBestDimension(Dictionary> pointChildren, Dictionary> anomalyChildren, List aggDim, double totalEntropy) + public BestDimension SelectBestDimension(Dictionary> pointChildren, Dictionary> anomalyChildren, List aggDim, double totalEntropy) { Dictionary entroyGainMap = new Dictionary(); Dictionary entroyGainRatioMap = new Dictionary(); @@ -453,7 +477,7 @@ public static BestDimension SelectBestDimension(Dictionary> return best; } - private static BestDimension FindBestDimension(Dictionary entropyGainMap, Dictionary entropyGainRationMap, double meanGain) + private BestDimension FindBestDimension(Dictionary entropyGainMap, Dictionary entropyGainRationMap, double meanGain) { BestDimension best = null; foreach (KeyValuePair dimension in entropyGainMap) @@ -486,7 +510,7 @@ private static BestDimension FindBestDimension(Dictionary return best; } - public static Point FindPointByDimension(Dictionary dim, List points) + public Point FindPointByDimension(Dictionary dim, List points) { foreach (Point p in points) { @@ -508,11 +532,11 @@ public static Point FindPointByDimension(Dictionary dim, List points, ref RootCause dst) + public void UpdateRootCauseDirection(List points, ref RootCause dst) { foreach (RootCauseItem item in dst.Items) { - Point rootCausePoint = DTRootCauseAnalyzer.FindPointByDimension(item.Dimension, points); + Point rootCausePoint = FindPointByDimension(item.Dimension, points); if (rootCausePoint != null) { if (rootCausePoint.ExpectedValue < rootCausePoint.Value) @@ -528,12 +552,12 @@ public static void UpdateRootCauseDirection(List points, ref RootCause ds } } - public static void GetRootCauseScore(List points, Dictionary anomalyRoot, ref RootCause dst, double beta) + public void GetRootCauseScore(List points, Dictionary anomalyRoot, ref RootCause dst, double beta) { if (dst.Items.Count > 1) { //get surprise value and explanary power value - Point anomalyPoint = DTRootCauseAnalyzer.FindPointByDimension(anomalyRoot, points); + Point anomalyPoint = FindPointByDimension(anomalyRoot, points); double sumSurprise = 0; double sumEp = 0; @@ -541,7 +565,7 @@ public static void GetRootCauseScore(List points, Dictionary scores = GetSupriseAndExplainaryScore(rootCausePoint, anomalyPoint); @@ -554,40 +578,40 @@ public static void GetRootCauseScore(List points, Dictionary scores = GetSupriseAndExplainaryScore(rootCausePoint, anomalyPoint); - dst.Items[0].Score = DTRootCauseAnalyzer.GetFinalScore(scores.Item1, scores.Item2, beta); + dst.Items[0].Score = GetFinalScore(scores.Item1, scores.Item2, beta); } } } - private static double GetSurpriseScore(Point rootCausePoint, Point anomalyPoint) + private double GetSurpriseScore(Point rootCausePoint, Point anomalyPoint) { double p = rootCausePoint.ExpectedValue / anomalyPoint.ExpectedValue; double q = rootCausePoint.Value / anomalyPoint.Value; - double surprise = 0.5 * (p * DTRootCauseAnalyzer.Log2(2 * p / (p + q)) + q * DTRootCauseAnalyzer.Log2(2 * q / (p + q))); + double surprise = 0.5 * (p * Log2(2 * p / (p + q)) + q * Log2(2 * q / (p + q))); return surprise; } - private static double GetFinalScore(double surprise, double ep, double beta) + private double GetFinalScore(double surprise, double ep, double beta) { return Math.Max(1, beta * surprise + (1 - beta) * ep); } - private static Tuple GetSupriseAndExplainaryScore(Point rootCausePoint, Point anomalyPoint) + private Tuple GetSupriseAndExplainaryScore(Point rootCausePoint, Point anomalyPoint) { - double surprise = DTRootCauseAnalyzer.GetSurpriseScore(rootCausePoint, anomalyPoint); + double surprise = GetSurpriseScore(rootCausePoint, anomalyPoint); double ep = (rootCausePoint.Value - rootCausePoint.ExpectedValue) / (anomalyPoint.Value - anomalyPoint.ExpectedValue); return new Tuple(surprise, ep); @@ -599,7 +623,7 @@ private static Dictionary UpdateDimensionValue(Dictionary 2; } - private static bool ShouldSeperateAnomaly(double total, double parent, int totalSize, int size) + private bool ShouldSeperateAnomaly(double total, double parent, int totalSize, int size) { if (Math.Abs(total) < Math.Abs(parent) * 0.95) { @@ -624,7 +648,7 @@ private static bool ShouldSeperateAnomaly(double total, double parent, int total return size <= totalSize * 0.5; } - private static double GetDimensionEntropyGain(Dictionary pointDis, Dictionary anomalyDis, double totalEntropy) + private double GetDimensionEntropyGain(Dictionary pointDis, Dictionary anomalyDis, double totalEntropy) { int total = GetTotalNumber(pointDis); double entropy = 0; @@ -636,7 +660,7 @@ private static double GetDimensionEntropyGain(Dictionary pointDis, return totalEntropy - entropy; } - private static double GetDimensionInstrinsicValue(Dictionary pointDis, Dictionary anomalyDis, double totalEntropy) + private double GetDimensionInstrinsicValue(Dictionary pointDis, Dictionary anomalyDis, double totalEntropy) { double instrinsicValue = 0; @@ -648,7 +672,7 @@ private static double GetDimensionInstrinsicValue(Dictionary pointD return instrinsicValue; } - private static int GetTotalNumber(Dictionary distribution) + private int GetTotalNumber(Dictionary distribution) { int total = 0; foreach (int num in distribution.Values) @@ -658,7 +682,7 @@ private static int GetTotalNumber(Dictionary distribution) return total; } - private static void UpdateDistribution(Dictionary distribution, List points, string dimKey) + private void UpdateDistribution(Dictionary distribution, List points, string dimKey) { foreach (Point point in points) { @@ -671,12 +695,12 @@ private static void UpdateDistribution(Dictionary distribution, Lis } } - public static double Log2(double val) + public double Log2(double val) { return Math.Log(val) / Math.Log(2); } - public static bool ContainsAll(Dictionary bigDic, Dictionary smallDic) + public bool ContainsAll(Dictionary bigDic, Dictionary smallDic) { foreach (var item in smallDic) { @@ -693,12 +717,12 @@ public static bool ContainsAll(Dictionary bigDic, Dictionary Items { get; set; } + public RootCause() + { + Items = new List(); + } } public sealed class RootCauseLocalizationInput diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index 7efb0a5083..641a496c3b 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -769,16 +769,16 @@ private static DateTime GetCurrentTimestamp() return new DateTime(); Dictionary expectedDim = new Dictionary(); - expectedDim.Add("Country","UK"); - expectedDim.Add("DeviceType",_aggSymbol); - expectedDim.Add("DataCenter","DC1"); + expectedDim.Add("Country", "UK"); + expectedDim.Add("DeviceType", _aggSymbol); + expectedDim.Add("DataCenter", "DC1"); - foreach (KeyValuePair pair in transformedRootCause.RootCause.Items[0].Dimension) { + foreach (KeyValuePair pair in transformedRootCause.RootCause.Items[0].Dimension) + { Assert.Equal(expectedDim[pair.Key], pair.Value); } } - private static List GetRootCauseLocalizationPoints() { List points = new List(); @@ -852,8 +852,7 @@ private static Dictionary GetAnomalyDimension() private static DateTime GetCurrentTimestamp() { - return new DateTime(); + return new DateTime(2020, 3, 23, 0, 0, 0); } - } } From 612be4d5c762a99e6aa45a513fd5175730a368b3 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:48:38 +0800 Subject: [PATCH 12/36] merge --- .../DTRootCauseAnalyzer.cs | 929 ------------------ .../DTRootCauseLocalizationType.cs | 172 ---- 2 files changed, 1101 deletions(-) delete mode 100644 src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs delete mode 100644 src/Microsoft.ML.TimeSeries/DTRootCauseLocalizationType.cs diff --git a/src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs deleted file mode 100644 index c127b7067f..0000000000 --- a/src/Microsoft.ML.TimeSeries/DTRootCauseAnalyzer.cs +++ /dev/null @@ -1,929 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.ML.Internal.Utilities; - -namespace Microsoft.ML.TimeSeries -{ - public class DTRootCauseAnalyzer - { - private RootCauseLocalizationInput _src; - private double _beta; - public DTRootCauseAnalyzer(RootCauseLocalizationInput src, double beta) - { - _src = src; - _beta = beta; - } - - public RootCause Analyze() - { - RootCause dst = new RootCause(); - DimensionInfo dimensionInfo = SeperateDimension(_src.AnomalyDimensions, _src.AggSymbol); - if (dimensionInfo.AggDim.Count == 0) - { - dst.Items.Add(new RootCauseItem(_src.AnomalyDimensions)); - } - Dictionary subDim = GetSubDim(_src.AnomalyDimensions, dimensionInfo.DetailDim); - List totalPoints = GetTotalPointsForAnomalyTimestamp(_src, subDim); - - GetRootCauseList(_src, ref dst, dimensionInfo, totalPoints, subDim); - UpdateRootCauseDirection(totalPoints, ref dst); - return dst; - } - - public List GetTotalPointsForAnomalyTimestamp(RootCauseLocalizationInput src, Dictionary subDim) - { - List points = new List(); - foreach (MetricSlice slice in src.Slices) - { - if (slice.TimeStamp.Equals(src.AnomalyTimestamp)) - { - points = slice.Points; - } - } - - List totalPoints = SelectPoints(points, subDim); - - return totalPoints; - } - - public void GetRootCauseList(RootCauseLocalizationInput src, ref RootCause dst, DimensionInfo dimensionInfo, List totalPoints, Dictionary subDim) - { - PointTree pointTree = BuildPointTree(totalPoints, dimensionInfo.AggDim, subDim, src.AggSymbol, src.AggType); - PointTree anomalyTree = BuildPointTree(totalPoints, dimensionInfo.AggDim, subDim, src.AggSymbol, src.AggType, true); - - //which means there is no aggregation in the input anomaly dimension - if (anomalyTree.ParentNode == null) - { - return; - } - - List rootCauses = new List(); - // no point under anomaly dimension - if (totalPoints.Count == 0) - { - if (anomalyTree.Leaves.Count != 0) - { - throw new Exception("point leaves not match with anomaly leaves"); - } - - rootCauses.Add(new RootCauseItem(src.AnomalyDimensions)); - } - else - { - double totalEntropy = 1; - if (anomalyTree.Leaves.Count > 0) - { - totalEntropy = GetEntropy(pointTree.Leaves.Count, anomalyTree.Leaves.Count); - } - - if (totalEntropy > 0.9) - { - rootCauses.AddRange(LocalizeRootCauseByDimension(pointTree.Leaves, anomalyTree, pointTree, totalEntropy, src.AnomalyDimensions)); - } - else - { - rootCauses.AddRange(LocalizeRootCauseByDimension(pointTree.Leaves, anomalyTree, pointTree, totalEntropy, src.AnomalyDimensions)); - } - - dst.Items = rootCauses; - } - } - - public DimensionInfo SeperateDimension(Dictionary dimensions, string aggSymbol) - { - DimensionInfo info = DimensionInfo.CreateDefaultInstance(); - foreach (KeyValuePair entry in dimensions) - { - string key = entry.Key; - if (aggSymbol.Equals(entry.Value)) - { - info.AggDim.Add(key); - } - else - { - info.DetailDim.Add(key); - } - } - - return info; - } - - protected PointTree BuildPointTree(List pointList, List aggDims, Dictionary subDim, string aggSymbol, AggregateType aggType, bool filterByAnomaly = false) - { - PointTree tree = PointTree.CreateDefaultInstance(); - - foreach (Point point in pointList) - { - bool isValidPoint = true; - if (filterByAnomaly) - { - isValidPoint = point.IsAnomaly == true; - } - if (ContainsAll(point.Dimensions, subDim) && isValidPoint) - { - if (aggDims.Count == 0) - { - tree.ParentNode = point; - tree.Leaves.Add(point); - } - else - { - int aggNum = 0; - string nextDim = null; - - foreach (string dim in aggDims) - { - if (IsAggregationDimension(point.Dimensions[dim], aggSymbol)) - { - aggNum++; - } - else - { - nextDim = dim; - } - } - - if (aggNum == aggDims.Count) - { - tree.ParentNode = point; - } - else if (aggNum == aggDims.Count - 1) - { - if (!tree.ChildrenNodes.ContainsKey(nextDim)) - { - tree.ChildrenNodes.Add(nextDim, new List()); - } - tree.ChildrenNodes[nextDim].Add(point); - } - - if (aggNum == 0) - { - tree.Leaves.Add(point); - } - } - } - } - - return tree; - } - private PointTree CompleteTreeBottomUp(PointTree tree, AggregateType aggType, string aggSymbol, List aggDims) - { - - if (tree.Leaves.Count == 0) return tree; - - Dictionary> map = new Dictionary>(); - foreach (Point p in tree.Leaves) - { - foreach (KeyValuePair keyValuePair in p.Dimensions) - { - if (aggDims.Contains(keyValuePair.Key)) - { - if (map.ContainsKey(keyValuePair.Key)) - { - map[keyValuePair.Key].Add(keyValuePair.Value); - } - else - { - map.Add(keyValuePair.Key, new HashSet() { keyValuePair.Value }); - } - } - } - } - - foreach (KeyValuePair> pair in map) - { - if (tree.ChildrenNodes.ContainsKey(pair.Key)) - { - if (tree.ChildrenNodes[pair.Key].Count < pair.Value.Count) - { - foreach (string value in pair.Value) - { - if (!IsAggDimensionExisted(pair.Key, value, tree.ChildrenNodes[pair.Key])) - { - Point p = SimulateBottomUpValue(tree.Leaves, pair.Key, value, aggType, aggSymbol); - tree.ChildrenNodes[pair.Key].Add(p); - } - } - } - } - else - { - List childPoints = new List(); - foreach (string value in pair.Value) - { - //simulate the aggregation value - Point p = SimulateBottomUpValue(tree.Leaves, pair.Key, value, aggType, aggSymbol); - childPoints.Add(p); - } - - tree.ChildrenNodes.Add(pair.Key, childPoints); - } - } - - return tree; - } - - private bool IsAggDimensionExisted(string key, string value, List points) - { - foreach (Point p in points) - { - if (p.Dimensions[key].Equals(value)) - { - return true; - } - } - return false; - } - - private Point SimulateBottomUpValue(List leaves, string key, string keyValue, AggregateType type, string aggSymbol) - { - Point p = null; - - Dictionary dimension = new Dictionary(); - - dimension.Add(key, keyValue); - - foreach (KeyValuePair pair in leaves[0].Dimensions) - { - if (!pair.Key.Equals(key)) - { - dimension.Add(pair.Key, aggSymbol); - } - } - - if (type.Equals(AggregateType.Sum)) - { - - bool isAnomaly = false; - double value = 0; - double expectedValue = 0; - foreach (Point leave in leaves) - { - - if (leave.Dimensions.ContainsKey(key) && leave.Dimensions[key].Equals(keyValue)) - { - value += leave.Value; - expectedValue = leave.ExpectedValue; - isAnomaly = isAnomaly || leave.IsAnomaly; - } - } - - p = new Point(value, expectedValue, isAnomaly, dimension); - } - - return p; - } - - public Dictionary GetSubDim(Dictionary dimension, List keyList) - { - Dictionary subDim = new Dictionary(); - - foreach (String dim in keyList) - { - subDim.Add(dim, dimension[dim]); - } - return subDim; - } - - protected List SelectPoints(List points, Dictionary subDim) - { - List list = new List(); - - foreach (Point point in points) - { - if (ContainsAll(point.Dimensions, subDim)) - { - //remove duplicated points - if (!list.Contains(point)) - { - list.Add(point); - } - } - } - - return list; - } - - protected List LocalizeRootCauseByDimension(List totalPoints, PointTree anomalyTree, PointTree pointTree, double totoalEntropy, Dictionary anomalyDimension) - { - var set = anomalyTree.ChildrenNodes.Keys; - - BestDimension best = null; - if (anomalyTree.Leaves.Count > 0) - { - best = SelectBestDimension(totalPoints, anomalyTree.Leaves, set.ToList(), totoalEntropy); - } - else - { - //has no leaves information, should calculate the entropy information according to the children nodes - best = SelectBestDimension(pointTree.ChildrenNodes, anomalyTree.ChildrenNodes, set.ToList(), totoalEntropy); - } - - if (best == null) - { - return new List() { new RootCauseItem(anomalyDimension) }; - } - - List children = GetTopAnomaly(anomalyTree.ChildrenNodes[best.DimensionKey], anomalyTree.ParentNode, totalPoints, best.DimensionKey); - if (children == null) - { - //As the cause couldn't be found, the root cause should be itself - return new List() { new RootCauseItem(anomalyDimension, best.DimensionKey) }; - } - else - { - List causes = new List(); - // For the found causes, we return the result - foreach (Point anomaly in children) - { - causes.Add(new RootCauseItem(UpdateDimensionValue(anomalyDimension, best.DimensionKey, anomaly.Dimensions[best.DimensionKey]), best.DimensionKey)); - } - return causes; - } - } - - protected double GetEntropy(int totalNum, int anomalyNum) - { - double ratio = (double)anomalyNum / totalNum; - if (ratio == 0 || ratio == 1) - { - return 0; - } - - return -(ratio * Log2(ratio) + (1 - ratio) * Log2(1 - ratio)); - } - - protected Dictionary GetEntropyList(BestDimension best, List points) - { - Dictionary list = new Dictionary(); - // need to update, change to children if necessary - foreach (Point point in points) - { - string dimVal = point.Dimensions[best.DimensionKey]; - int pointSize = GetPointSize(best, dimVal); - int anomalySize = GetAnomalyPointSize(best, dimVal); - - double dimEntropy = GetEntropy(pointSize, anomalySize); - list.Add(dimVal, dimEntropy); - } - - return list; - } - - protected List GetTopAnomaly(List anomalyPoints, Point root, List totalPoints, string dimKey) - { - Dictionary pointDistribution = new Dictionary(); - UpdateDistribution(pointDistribution, totalPoints, dimKey); - - anomalyPoints.OrderBy(x => x.Delta); - - if (root.Delta > 0) - { - anomalyPoints.Reverse(); - } - - if (anomalyPoints.Count == 1) - { - return anomalyPoints; - } - - double delta = 0; - double preDelta = 0; - List causeList = new List(); - foreach (Point anomaly in anomalyPoints) - { - if (StopAnomalyComparison(delta, root.Delta, anomaly.Delta, preDelta)) - { - break; - } - - delta += anomaly.Delta; - causeList.Add(anomaly); - preDelta = anomaly.Delta; - } - - int pointSize = GetTotalNumber(pointDistribution); - if (ShouldSeperateAnomaly(delta, root.Delta, pointSize, causeList.Count)) - { - return causeList; - } - - return null; - } - - protected BestDimension SelectBestDimension(List totalPoints, List anomalyPoints, List aggDim, double totalEntropy) - { - Dictionary entroyGainMap = new Dictionary(); - Dictionary entroyGainRatioMap = new Dictionary(); - double sumGain = 0; - - foreach (string dimKey in aggDim) - { - BestDimension dimension = BestDimension.CreateDefaultInstance(); - dimension.DimensionKey = dimKey; - - UpdateDistribution(dimension.PointDis, totalPoints, dimKey); - UpdateDistribution(dimension.AnomalyDis, anomalyPoints, dimKey); - - double gain = GetDimensionEntropyGain(dimension.PointDis, dimension.AnomalyDis, totalEntropy); - dimension.Entropy = totalEntropy - gain; - entroyGainMap.Add(dimension, gain); - - double gainRatio = gain / GetDimensionInstrinsicValue(dimension.PointDis, dimension.AnomalyDis, totalEntropy); - entroyGainRatioMap.Add(dimension, gainRatio); - - sumGain += gain; - } - - double meanGain = sumGain / aggDim.Count(); - - BestDimension best = FindBestDimension(entroyGainMap, entroyGainRatioMap, meanGain); - return best; - } - - public BestDimension SelectBestDimension(Dictionary> pointChildren, Dictionary> anomalyChildren, List aggDim, double totalEntropy) - { - Dictionary entroyGainMap = new Dictionary(); - Dictionary entroyGainRatioMap = new Dictionary(); - double sumGain = 0; - - foreach (string dimKey in aggDim) - { - BestDimension dimension = BestDimension.CreateDefaultInstance(); - dimension.DimensionKey = dimKey; - - UpdateDistribution(dimension.PointDis, pointChildren[dimKey], dimKey); - UpdateDistribution(dimension.AnomalyDis, anomalyChildren[dimKey], dimKey); - - double gain = GetDimensionEntropyGain(dimension.PointDis, dimension.AnomalyDis, totalEntropy); - dimension.Entropy = totalEntropy - gain; - entroyGainMap.Add(dimension, gain); - - double gainRatio = gain / GetDimensionInstrinsicValue(dimension.PointDis, dimension.AnomalyDis, totalEntropy); - entroyGainRatioMap.Add(dimension, gainRatio); - - sumGain += gain; - } - - double meanGain = sumGain / aggDim.Count(); - - BestDimension best = FindBestDimension(entroyGainMap, entroyGainRatioMap, meanGain); - - return best; - } - - private BestDimension FindBestDimension(Dictionary entropyGainMap, Dictionary entropyGainRationMap, double meanGain) - { - BestDimension best = null; - foreach (KeyValuePair dimension in entropyGainMap) - { - if (dimension.Key.AnomalyDis.Count == 1 || dimension.Value >= meanGain) - { - if (dimension.Key.AnomalyDis.Count > 1) - { - if (best == null || (best.AnomalyDis.Count != 1 && entropyGainRationMap[best].CompareTo(dimension.Value) < 0)) - { - best = dimension.Key; - } - } - else - { - if (best == null || best.AnomalyDis.Count > 1) - { - best = dimension.Key; - } - else - { - if (entropyGainRationMap[best].CompareTo(dimension.Value) < 0) - { - best = dimension.Key; - } - } - } - } - } - return best; - } - - public Point FindPointByDimension(Dictionary dim, List points) - { - foreach (Point p in points) - { - bool isEqual = true; - foreach (KeyValuePair item in p.Dimensions) - { - if (!dim[item.Key].Equals(item.Value)) - { - isEqual = false; - } - } - - if (isEqual) - { - return p; - } - } - - return null; - } - - public void UpdateRootCauseDirection(List points, ref RootCause dst) - { - foreach (RootCauseItem item in dst.Items) - { - Point rootCausePoint = FindPointByDimension(item.Dimension, points); - if (rootCausePoint != null) - { - if (rootCausePoint.ExpectedValue < rootCausePoint.Value) - { - item.Direction = AnomalyDirection.Up; - } - else - { - item.Direction = AnomalyDirection.Down; - } - } - - } - } - - public void GetRootCauseScore(List points, Dictionary anomalyRoot, ref RootCause dst, double beta) - { - if (dst.Items.Count > 1) - { - //get surprise value and explanary power value - Point anomalyPoint = FindPointByDimension(anomalyRoot, points); - - double sumSurprise = 0; - double sumEp = 0; - List scoreList = new List(); - - foreach (RootCauseItem item in dst.Items) - { - Point rootCausePoint = FindPointByDimension(item.Dimension, points); - if (anomalyPoint != null && rootCausePoint != null) - { - Tuple scores = GetSupriseAndExplainaryScore(rootCausePoint, anomalyPoint); - scoreList.Add(new RootCauseScore(scores.Item1, scores.Item2)); - sumSurprise += scores.Item1; - sumEp += Math.Abs(scores.Item2); - } - } - - //normalize and get final score - for (int i = 0; i < scoreList.Count; i++) - { - dst.Items[i].Score = GetFinalScore(scoreList[i].Surprise / sumSurprise, Math.Abs(scoreList[i].ExplainaryScore) / sumEp, beta); - - } - } - else if (dst.Items.Count == 1) - { - Point rootCausePoint = FindPointByDimension(dst.Items[0].Dimension, points); - - Point anomalyPoint = FindPointByDimension(anomalyRoot, points); - if (anomalyPoint != null && rootCausePoint != null) - { - Tuple scores = GetSupriseAndExplainaryScore(rootCausePoint, anomalyPoint); - dst.Items[0].Score = GetFinalScore(scores.Item1, scores.Item2, beta); - } - } - } - - private double GetSurpriseScore(Point rootCausePoint, Point anomalyPoint) - { - double p = rootCausePoint.ExpectedValue / anomalyPoint.ExpectedValue; - double q = rootCausePoint.Value / anomalyPoint.Value; - double surprise = 0.5 * (p * Log2(2 * p / (p + q)) + q * Log2(2 * q / (p + q))); - - return surprise; - } - - private double GetFinalScore(double surprise, double ep, double beta) - { - return Math.Max(1, beta * surprise + (1 - beta) * ep); - } - - private Tuple GetSupriseAndExplainaryScore(Point rootCausePoint, Point anomalyPoint) - { - double surprise = GetSurpriseScore(rootCausePoint, anomalyPoint); - double ep = (rootCausePoint.Value - rootCausePoint.ExpectedValue) / (anomalyPoint.Value - anomalyPoint.ExpectedValue); - - return new Tuple(surprise, ep); - } - private static Dictionary UpdateDimensionValue(Dictionary dimension, string key, string value) - { - Dictionary newDim = new Dictionary(dimension); - newDim[key] = value; - return newDim; - } - - private bool StopAnomalyComparison(double preTotal, double parent, double current, double pre) - { - if (Math.Abs(preTotal) < Math.Abs(parent) * 0.95) - { - return false; - } - - return Math.Abs(pre) / Math.Abs(current) > 2; - } - - private bool ShouldSeperateAnomaly(double total, double parent, int totalSize, int size) - { - if (Math.Abs(total) < Math.Abs(parent) * 0.95) - { - return false; - } - - if (size == totalSize && size == 1) - { - return true; - } - - return size <= totalSize * 0.5; - } - - private double GetDimensionEntropyGain(Dictionary pointDis, Dictionary anomalyDis, double totalEntropy) - { - int total = GetTotalNumber(pointDis); - double entropy = 0; - foreach (string key in anomalyDis.Keys) - { - double dimEntropy = GetEntropy(pointDis[key], anomalyDis[key]); - entropy += dimEntropy * pointDis[key] / total; - } - return totalEntropy - entropy; - } - - private double GetDimensionInstrinsicValue(Dictionary pointDis, Dictionary anomalyDis, double totalEntropy) - { - double instrinsicValue = 0; - - foreach (string key in anomalyDis.Keys) - { - instrinsicValue -= Log2((double)anomalyDis[key] / pointDis[key]) * anomalyDis[key] / pointDis[key]; - } - - return instrinsicValue; - } - - private int GetTotalNumber(Dictionary distribution) - { - int total = 0; - foreach (int num in distribution.Values) - { - total += num; - } - return total; - } - - private void UpdateDistribution(Dictionary distribution, List points, string dimKey) - { - foreach (Point point in points) - { - string dimVal = point.Dimensions[dimKey]; - if (!distribution.ContainsKey(dimVal)) - { - distribution.Add(dimVal, 0); - } - distribution[dimVal] = distribution[dimVal] + 1; - } - } - - public double Log2(double val) - { - return Math.Log(val) / Math.Log(2); - } - - public bool ContainsAll(Dictionary bigDic, Dictionary smallDic) - { - foreach (var item in smallDic) - { - if (!bigDic.ContainsKey(item.Key)) - { - return false; - } - - if (bigDic.ContainsKey(item.Key) && !bigDic[item.Key].Equals(smallDic[item.Key])) - { - return false; - } - } - return true; - } - - private bool IsAggregationDimension(string val, string aggSymbol) - { - return val.Equals(aggSymbol); - } - - private int GetPointSize(BestDimension dim, string key) - { - int pointSize = 0; - if (dim.PointDis.ContainsKey(key)) - { - pointSize = dim.PointDis[key]; - } - return pointSize; - } - - private int GetAnomalyPointSize(BestDimension dim, string key) - { - int anomalyPointSize = 0; - if (dim.AnomalyDis.ContainsKey(key)) - { - anomalyPointSize = dim.AnomalyDis[key]; - } - return anomalyPointSize; - } - } - - public class DimensionInfo - { - public List DetailDim { get; set; } - public List AggDim { get; set; } - - public static DimensionInfo CreateDefaultInstance() - { - DimensionInfo instance = new DimensionInfo(); - instance.DetailDim = new List(); - instance.AggDim = new List(); - return instance; - } - } - - public class PointTree - { - public Point ParentNode; - public Dictionary> ChildrenNodes; - public List Leaves; - - public static PointTree CreateDefaultInstance() - { - PointTree instance = new PointTree(); - instance.Leaves = new List(); - instance.ChildrenNodes = new Dictionary>(); - return instance; - } - } - - public sealed class Point : IEquatable - { - public double Value { get; set; } - public double ExpectedValue { get; set; } - public bool IsAnomaly { get; set; } - public Dictionary Dimensions { get; set; } - - public double Delta { get; set; } - - public Point(double value, double expectedValue, bool isAnomaly, Dictionary dimensions) - { - Value = value; - ExpectedValue = expectedValue; - IsAnomaly = isAnomaly; - Dimensions = dimensions; - Delta = (value - expectedValue) / expectedValue; - if (expectedValue == 0) - { - Delta = 0; - } - } - - public bool Equals(Point other) - { - foreach (KeyValuePair item in Dimensions) - { - if (!other.Dimensions[item.Key].Equals(item.Value)) - { - return false; - } - } - return true; - } - - public override int GetHashCode() - { - return Dimensions.GetHashCode(); - } - } - - public sealed class MetricSlice - { - public DateTime TimeStamp { get; set; } - public List Points { get; set; } - - public MetricSlice(DateTime timeStamp, List points) - { - TimeStamp = timeStamp; - Points = points; - } - } - - public sealed class BestDimension - { - public string DimensionKey; - public double Entropy; - public Dictionary AnomalyDis; - public Dictionary PointDis; - - public BestDimension() { } - public static BestDimension CreateDefaultInstance() - { - BestDimension instance = new BestDimension(); - instance.AnomalyDis = new Dictionary(); - instance.PointDis = new Dictionary(); - return instance; - } - } - - public sealed class AnomalyCause - { - public string DimensionKey; - public List Anomalies; - - public AnomalyCause() { } - } - - public sealed class RootCauseItem : IEquatable - { - public double Score; - public string Path; - public Dictionary Dimension; - public AnomalyDirection Direction; - - public RootCauseItem(Dictionary rootCause) - { - Dimension = rootCause; - } - - public RootCauseItem(Dictionary rootCause, string path) - { - Dimension = rootCause; - Path = path; - } - public bool Equals(RootCauseItem other) - { - if (Dimension.Count == other.Dimension.Count) - { - foreach (KeyValuePair item in Dimension) - { - if (!other.Dimension[item.Key].Equals(item.Value)) - { - return false; - } - } - return true; - } - return false; - } - } - - public enum AnomalyDirection - { - /// - /// the value is larger than expected value. - /// - Up = 0, - /// - /// the value is lower than expected value. - /// - Down = 1 - } - - public class RootCauseScore - { - public double Surprise; - public double ExplainaryScore; - - public RootCauseScore(double surprise, double explainaryScore) - { - Surprise = surprise; - ExplainaryScore = explainaryScore; - } - } - - public enum AggregateType - { - /// - /// Make the aggregate type as sum. - /// - Sum = 0, - /// - /// Make the aggregate type as average. - /// - Avg = 1, - /// - /// Make the aggregate type as min. - /// - Min = 2, - /// - /// Make the aggregate type as max. - /// - Max = 3 - } -} diff --git a/src/Microsoft.ML.TimeSeries/DTRootCauseLocalizationType.cs b/src/Microsoft.ML.TimeSeries/DTRootCauseLocalizationType.cs deleted file mode 100644 index 4ecd41f17b..0000000000 --- a/src/Microsoft.ML.TimeSeries/DTRootCauseLocalizationType.cs +++ /dev/null @@ -1,172 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using Microsoft.ML.Data; - -namespace Microsoft.ML.TimeSeries -{ - /// - /// Allows a member to be marked as a , primarily allowing one to set - /// root cause localization input. - /// - public sealed class RootCauseLocalizationInputTypeAttribute : DataViewTypeAttribute - { - /// - /// Create a root cause localizagin input type. - /// - public RootCauseLocalizationInputTypeAttribute() - { - } - - /// - /// Equal function. - /// - public override bool Equals(DataViewTypeAttribute other) - { - if (!(other is RootCauseLocalizationInputTypeAttribute otherAttribute)) - return false; - return true; - } - - /// - /// Produce the same hash code for all RootCauseLocalizationInputTypeAttribute. - /// - public override int GetHashCode() - { - return 0; - } - - public override void Register() - { - DataViewTypeManager.Register(new RootCauseLocalizationInputDataViewType(), typeof(RootCauseLocalizationInput), this); - } - } - - /// - /// Allows a member to be marked as a , primarily allowing one to set - /// root cause result. - /// - public sealed class RootCauseTypeAttribute : DataViewTypeAttribute - { - /// - /// Create an root cause type. - /// - public RootCauseTypeAttribute() - { - } - - /// - /// RootCauseTypeAttribute with the same type should equal. - /// - public override bool Equals(DataViewTypeAttribute other) - { - if (other is RootCauseTypeAttribute otherAttribute) - return true; - return false; - } - - /// - /// Produce the same hash code for all RootCauseTypeAttribute. - /// - public override int GetHashCode() - { - return 0; - } - - public override void Register() - { - DataViewTypeManager.Register(new RootCauseDataViewType(), typeof(RootCause), this); - } - } - - public sealed class RootCause - { - public List Items { get; set; } - public RootCause() - { - Items = new List(); - } - } - - public sealed class RootCauseLocalizationInput - { - public DateTime AnomalyTimestamp { get; set; } - - public Dictionary AnomalyDimensions { get; set; } - - public List Slices { get; set; } - - public AggregateType AggType { get; set; } - - public string AggSymbol { get; set; } - - public RootCauseLocalizationInput(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, AggregateType aggregateType, string aggregateSymbol) - { - AnomalyTimestamp = anomalyTimestamp; - AnomalyDimensions = anomalyDimensions; - Slices = slices; - AggType = aggregateType; - AggSymbol = aggregateSymbol; - } - public void Dispose() - { - AnomalyDimensions = null; - Slices = null; - } - } - - public sealed class RootCauseDataViewType : StructuredDataViewType - { - public RootCauseDataViewType() - : base(typeof(RootCause)) - { - } - - public override bool Equals(DataViewType other) - { - if (other == this) - return true; - if (!(other is RootCauseDataViewType tmp)) - return false; - return true; - } - - public override int GetHashCode() - { - return 0; - } - - public override string ToString() - { - return typeof(RootCauseDataViewType).Name; - } - } - - public sealed class RootCauseLocalizationInputDataViewType : StructuredDataViewType - { - public RootCauseLocalizationInputDataViewType() - : base(typeof(RootCauseLocalizationInput)) - { - } - - public override bool Equals(DataViewType other) - { - if (!(other is RootCauseLocalizationInputDataViewType tmp)) - return false; - return true; - } - - public override int GetHashCode() - { - return 0; - } - - public override string ToString() - { - return typeof(RootCauseLocalizationInputDataViewType).Name; - } - } -} From 23261f2055455e696086cadd4435595339e2b929 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 15:49:53 +0800 Subject: [PATCH 13/36] merge --- .../TimeSeriesDirectApi.cs | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index 641a496c3b..22220fc844 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -854,5 +854,81 @@ private static DateTime GetCurrentTimestamp() { return new DateTime(2020, 3, 23, 0, 0, 0); } + + private static List GetRootCauseLocalizationPoints() + { + List points = new List(); + + Dictionary dic1 = new Dictionary(); + dic1.Add("Country", "UK"); + dic1.Add("DeviceType", "Laptop"); + dic1.Add("DataCenter", "DC1"); + points.Add(new Point(200, 100, true, dic1)); + + Dictionary dic2 = new Dictionary(); + dic2.Add("Country", "UK"); + dic2.Add("DeviceType", "Mobile"); + dic2.Add("DataCenter", "DC1"); + points.Add(new Point(1000, 100, true, dic2)); + + Dictionary dic3 = new Dictionary(); + dic3.Add("Country", "UK"); + dic3.Add("DeviceType", _aggSymbol); + dic3.Add("DataCenter", "DC1"); + points.Add(new Point(1200, 200, true, dic3)); + + Dictionary dic4 = new Dictionary(); + dic4.Add("Country", "UK"); + dic4.Add("DeviceType", "Laptop"); + dic4.Add("DataCenter", "DC2"); + points.Add(new Point(100, 100, false, dic4)); + + Dictionary dic5 = new Dictionary(); + dic5.Add("Country", "UK"); + dic5.Add("DeviceType", "Mobile"); + dic5.Add("DataCenter", "DC2"); + points.Add(new Point(200, 200, false, dic5)); + + Dictionary dic6 = new Dictionary(); + dic6.Add("Country", "UK"); + dic6.Add("DeviceType", _aggSymbol); + dic6.Add("DataCenter", "DC2"); + points.Add(new Point(300, 300, false, dic6)); + + Dictionary dic7 = new Dictionary(); + dic7.Add("Country", "UK"); + dic7.Add("DeviceType", _aggSymbol); + dic7.Add("DataCenter", _aggSymbol); + points.Add(new Point(1500, 500, true, dic7)); + + Dictionary dic8 = new Dictionary(); + dic8.Add("Country", "UK"); + dic8.Add("DeviceType", "Laptop"); + dic8.Add("DataCenter", _aggSymbol); + points.Add(new Point(300, 200, true, dic8)); + + Dictionary dic9 = new Dictionary(); + dic9.Add("Country", "UK"); + dic9.Add("DeviceType", "Mobile"); + dic9.Add("DataCenter", _aggSymbol); + points.Add(new Point(1200, 300, true, dic9)); + + return points; + } + + private static Dictionary GetAnomalyDimension() + { + Dictionary dim = new Dictionary(); + dim.Add("Country", "UK"); + dim.Add("DeviceType", _aggSymbol); + dim.Add("DataCenter", _aggSymbol); + + return dim; + } + + private static DateTime GetCurrentTimestamp() + { + return new DateTime(2020, 3, 23, 0, 0, 0); + } } } From fa10bff0be8719e365d60f06a764a292f51e3784 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 16:02:38 +0800 Subject: [PATCH 14/36] merge --- .../time-series-root-cause-localization-dt.md | 47 +++++++++++++++++++ .../Microsoft.ML.Samples.csproj | 3 +- docs/samples/Microsoft.ML.Samples/Program.cs | 16 +++---- .../ExtensionsCatalog.cs | 2 +- .../TimeSeriesDirectApi.cs | 33 ++++++++++--- 5 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 docs/api-reference/time-series-root-cause-localization-dt.md diff --git a/docs/api-reference/time-series-root-cause-localization-dt.md b/docs/api-reference/time-series-root-cause-localization-dt.md new file mode 100644 index 0000000000..16994b8c48 --- /dev/null +++ b/docs/api-reference/time-series-root-cause-localization-dt.md @@ -0,0 +1,47 @@ +At Mircosoft, we develop a decision tree based root cause localization method which helps to find out the root causes for an anomaly incident incrementally. + +## Multi-Dimensional Root Cause Localization +It's a common case that one measure are collected with many dimensions (*e.g.*, Province, ISP) whose values are categorical(*e.g.*, Beijing or Shanghai for dimension Province). When a measure's value deviates from its expected value, this measure encounters anomalies. In such case, operators would like to localize the root cause dimension combinations rapidly and accurately. Multi-dimensional root cause localization is critical to troubleshoot and mitigate such case. + +## Algorithm + +The decision based root cause localization method is unsupervised, which means training step is no needed. It consists of the following major steps: +(1) Find best dimension which divides the anomaly and unanomaly data based on decision tree according to entropy gain and entropy gain ratio. +(2) Find the top anomaly points for the selected best dimension. + +### Decision Tree + +[Decision tree](https://en.wikipedia.org/wiki/Decision_tree) algorithm chooses the highest information gain to split or construct a decision tree.  We use it to choose the dimension which contributes the most to the anomaly. Following are some concepts used in decision tree. + +#### Information Entropy + +Information [entropy](https://en.wikipedia.org/wiki/Entropy_(information_theory)) is a measure of disorder or uncertainty. You can think of it as a measure of purity as well.The less the value , the more pure of data D. + +$$Ent(D) = - \sum_{k=1}^{|y|} p_k\log_2(p_k) $$ + +where $p_k$ represents the probability of an element in dataset. In our case, there are only two classed, the anomaly points and the normaly points. $|y|$ is the count of total anomalies. + +#### Information Gain +[Information gain](https://en.wikipedia.org/wiki/Information_gain_in_decision_trees) is a metric to measure the reduction of this disorder in our target class given additional information about it. Mathematically it can be written as: + +$$Gain(D, a) = Ent(D) - \sum_{v=1}^{|V|} \frac{|D^V|}{|D |} Ent(D^v) $$ + +Where $Ent(D^v)$ is the entropy of set points in D for which dimension $a$ is equal to $v$, $|D|$ is the total number of points in dataset $D$. $|D^V|$ is the total number of points in dataset $D$ for which dimension $a$ is equal to $v$. + +For all aggregated dimensions, we calculate the information for each dimension. The greater the reduction in this uncertainty, the more information is gained about D from dimension $a$. + +#### Entropy Gain Ratio + +Information gain is biased toward variables with large number of distinct values. A modification is [information gain ratio](https://en.wikipedia.org/wiki/Information_gain_ratio), which reduces its bias. + +$$Ratio(D, a) = \frac{Gain(D,a)} {IV(a)} $$ + +where intrinsic value(IV) is the entropy of split (with respect to dimension $a$ on focus). + +$$IV(a) = -\sum_{v=1}^V\frac{|D^v|} {|D|} \log_2 \frac{|D^v|} {|D|} $$ + +In out strategy, firstly, for all the aggration dimensions, we loop all the dimensions to find the dimension who's entropy gain is above mean entropy gain ration, then from the filtered dimensions, we select the dimension with highest entropy ratio as the best dimension. In the meanwhile, dimensions for which the anomaly value count is only one, we include it when calculation. + +> [!Note] +> 1. As our algorithm depends on the data you input, so if the input points is incorrect or incomplete, the calculated result will be unexpected. +> 2. Currently, the algorithm localize the root cause incrementally, which means at most one dimension with the values are detected. If you want to find out all the dimension that contributes to the anomaly, you can call this API recursively. diff --git a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj index 42f11558ea..5951c4bbd1 100644 --- a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj +++ b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj @@ -1,8 +1,7 @@  - - netcoreapp3.0 + netcoreapp2.1 Exe false diff --git a/docs/samples/Microsoft.ML.Samples/Program.cs b/docs/samples/Microsoft.ML.Samples/Program.cs index 930ccc57f3..6e65499862 100644 --- a/docs/samples/Microsoft.ML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.Samples/Program.cs @@ -13,17 +13,13 @@ internal static void RunAll() int samples = 0; foreach (var type in Assembly.GetExecutingAssembly().GetTypes()) { - if (type.Name.Equals("LocalizeRootCauseEvaluation")) - //if (type.Name.Equals("LocalizeRootCauseByDT")) - { - var sample = type.GetMethod("Example", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); + var sample = type.GetMethod("Example", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); - if (sample != null) - { - Console.WriteLine(type.Name); - sample.Invoke(null, null); - samples++; - } + if (sample != null) + { + Console.WriteLine(type.Name); + sample.Invoke(null, null); + samples++; } } diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 4739cec3ad..45d21f2a6f 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -226,7 +226,7 @@ private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocaliza } /// - /// Create , which localizes root causess using decision tree algorithm. + /// Create , which localizes root causes using decision tree algorithm. /// /// The transform's catalog. /// Name of the column resulting from the transformation of . diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index 22220fc844..ef96fb1eb2 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -101,6 +101,33 @@ private sealed class SrCnnAnomalyDetection private static Object _rootCauseAggSymbol = "##SUM##"; + private class RootCauseLocalizationData + { + [RootCauseLocalizationInputType] + public RootCauseLocalizationInput Input { get; set; } + + public RootCauseLocalizationData() + { + Input = null; + } + + public RootCauseLocalizationData(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, AggregateType aggregateteType, string aggregateSymbol) + { + Input = new RootCauseLocalizationInput(anomalyTimestamp, anomalyDimensions, slices, aggregateteType, aggregateSymbol); + } + } + + private class RootCauseLocalizationTransformedData + { + [RootCauseType()] + public RootCause RootCause { get; set; } + + public RootCauseLocalizationTransformedData() + { + RootCause = null; + } + } + [Fact] public void ChangeDetection() { @@ -581,18 +608,15 @@ public void TestSrCnnBatchAnomalyDetector( [CombinatorialValues(SrCnnDetectMode.AnomalyOnly, SrCnnDetectMode.AnomalyAndExpectedValue, SrCnnDetectMode.AnomalyAndMargin)]SrCnnDetectMode mode, [CombinatorialValues(true, false)]bool loadDataFromFile, [CombinatorialValues(-1, 24, 26, 512)]int batchSize) - { var ml = new MLContext(1); IDataView dataView; if (loadDataFromFile) { var dataPath = GetDataPath("Timeseries", "anomaly_detection.csv"); - // Load data from file into the dataView dataView = ml.Data.LoadFromTextFile(dataPath, hasHeader: true); } else - { // Generate sample series data with an anomaly var data = new List(); for (int index = 0; index < 20; index++) @@ -608,11 +632,9 @@ public void TestSrCnnBatchAnomalyDetector( // Convert data to IDataView. dataView = ml.Data.LoadFromEnumerable(data); } - // Setup the detection arguments string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction); string inputColumnName = nameof(TimeSeriesDataDouble.Value); - // Do batch anomaly detection var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName, threshold: 0.35, batchSize: batchSize, sensitivity: 90.0, mode); @@ -624,7 +646,6 @@ public void TestSrCnnBatchAnomalyDetector( int k = 0; foreach (var prediction in predictionColumn) - { switch (mode) { case SrCnnDetectMode.AnomalyOnly: From 919ed6b8b10735bcd723cdc9064f94c24f92641a Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 1 Apr 2020 21:11:24 +0800 Subject: [PATCH 15/36] update --- docs/samples/Microsoft.ML.Samples/Program.cs | 1 - src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Program.cs b/docs/samples/Microsoft.ML.Samples/Program.cs index 6e65499862..4c46399421 100644 --- a/docs/samples/Microsoft.ML.Samples/Program.cs +++ b/docs/samples/Microsoft.ML.Samples/Program.cs @@ -23,7 +23,6 @@ internal static void RunAll() } } - Console.WriteLine("Number of samples that ran without any exception: " + samples); } } diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 45d21f2a6f..8f8c681203 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -239,8 +239,8 @@ private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocaliza /// ]]> /// /// - public static DTRootCauseLocalizationEstimator LocalizeRootCauseByDT(this TransformsCatalog catalog, string outputColumnName, string inputColumnName = null, double beta=0.5) - => new DTRootCauseLocalizationEstimator(CatalogUtils.GetEnvironment(catalog),beta, new[] { (outputColumnName, inputColumnName ?? outputColumnName) }); + public static DTRootCauseLocalizationEstimator LocalizeRootCauseByDT(this TransformsCatalog catalog, string outputColumnName, string inputColumnName = null, double beta = 0.5) + => new DTRootCauseLocalizationEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, inputColumnName ?? outputColumnName, beta); /// /// Singular Spectrum Analysis (SSA) model for univariate time-series forecasting. From 04072828d48625e8a4dc464f61dab66e85f5425a Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Tue, 7 Apr 2020 16:50:22 +0800 Subject: [PATCH 16/36] refine internal logic --- .../ExtensionsCatalog.cs | 6 ++-- .../TimeSeriesDirectApi.cs | 29 ++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 8f8c681203..bc2b833218 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -229,8 +229,10 @@ private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocaliza /// Create , which localizes root causes using decision tree algorithm. /// /// The transform's catalog. - /// Name of the column resulting from the transformation of . - /// Name of column to transform. + /// Name of the column resulting from the transformation of . + /// The column data is an instance of . + /// Name of the input column. + /// The column data is an instance of . /// The weight parameter in score. The range of the parameter should be in [0,1]. /// /// diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index ef96fb1eb2..a2a95b9daf 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -8,6 +8,7 @@ using Microsoft.ML.Data; using Microsoft.ML.TestFramework; using Microsoft.ML.TimeSeries; +using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.TimeSeries; using Microsoft.ML.Transforms.TimeSeries; using Xunit; @@ -111,7 +112,7 @@ public RootCauseLocalizationData() Input = null; } - public RootCauseLocalizationData(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, AggregateType aggregateteType, string aggregateSymbol) + public RootCauseLocalizationData(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, AggregateType aggregateteType, string aggregateSymbol) { Input = new RootCauseLocalizationInput(anomalyTimestamp, anomalyDimensions, slices, aggregateteType, aggregateSymbol); } @@ -789,12 +790,12 @@ private static DateTime GetCurrentTimestamp() { return new DateTime(); - Dictionary expectedDim = new Dictionary(); + Dictionary expectedDim = new Dictionary(); expectedDim.Add("Country", "UK"); expectedDim.Add("DeviceType", _aggSymbol); expectedDim.Add("DataCenter", "DC1"); - foreach (KeyValuePair pair in transformedRootCause.RootCause.Items[0].Dimension) + foreach (KeyValuePair pair in transformedRootCause.RootCause.Items[0].Dimension) { Assert.Equal(expectedDim[pair.Key], pair.Value); } @@ -880,55 +881,55 @@ private static List GetRootCauseLocalizationPoints() { List points = new List(); - Dictionary dic1 = new Dictionary(); + Dictionary dic1 = new Dictionary(); dic1.Add("Country", "UK"); dic1.Add("DeviceType", "Laptop"); dic1.Add("DataCenter", "DC1"); points.Add(new Point(200, 100, true, dic1)); - Dictionary dic2 = new Dictionary(); + Dictionary dic2 = new Dictionary(); dic2.Add("Country", "UK"); dic2.Add("DeviceType", "Mobile"); dic2.Add("DataCenter", "DC1"); points.Add(new Point(1000, 100, true, dic2)); - Dictionary dic3 = new Dictionary(); + Dictionary dic3 = new Dictionary(); dic3.Add("Country", "UK"); dic3.Add("DeviceType", _aggSymbol); dic3.Add("DataCenter", "DC1"); points.Add(new Point(1200, 200, true, dic3)); - Dictionary dic4 = new Dictionary(); + Dictionary dic4 = new Dictionary(); dic4.Add("Country", "UK"); dic4.Add("DeviceType", "Laptop"); dic4.Add("DataCenter", "DC2"); points.Add(new Point(100, 100, false, dic4)); - Dictionary dic5 = new Dictionary(); + Dictionary dic5 = new Dictionary(); dic5.Add("Country", "UK"); dic5.Add("DeviceType", "Mobile"); dic5.Add("DataCenter", "DC2"); points.Add(new Point(200, 200, false, dic5)); - Dictionary dic6 = new Dictionary(); + Dictionary dic6 = new Dictionary(); dic6.Add("Country", "UK"); dic6.Add("DeviceType", _aggSymbol); dic6.Add("DataCenter", "DC2"); points.Add(new Point(300, 300, false, dic6)); - Dictionary dic7 = new Dictionary(); + Dictionary dic7 = new Dictionary(); dic7.Add("Country", "UK"); dic7.Add("DeviceType", _aggSymbol); dic7.Add("DataCenter", _aggSymbol); points.Add(new Point(1500, 500, true, dic7)); - Dictionary dic8 = new Dictionary(); + Dictionary dic8 = new Dictionary(); dic8.Add("Country", "UK"); dic8.Add("DeviceType", "Laptop"); dic8.Add("DataCenter", _aggSymbol); points.Add(new Point(300, 200, true, dic8)); - Dictionary dic9 = new Dictionary(); + Dictionary dic9 = new Dictionary(); dic9.Add("Country", "UK"); dic9.Add("DeviceType", "Mobile"); dic9.Add("DataCenter", _aggSymbol); @@ -937,9 +938,9 @@ private static List GetRootCauseLocalizationPoints() return points; } - private static Dictionary GetAnomalyDimension() + private static Dictionary GetAnomalyDimension() { - Dictionary dim = new Dictionary(); + Dictionary dim = new Dictionary(); dim.Add("Country", "UK"); dim.Add("DeviceType", _aggSymbol); dim.Add("DataCenter", _aggSymbol); From 0efee95950ef69d48aec9bd04520301847ef065b Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 16:47:07 +0800 Subject: [PATCH 17/36] merge --- test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index a2a95b9daf..188cddd05c 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -870,6 +870,14 @@ private static Dictionary GetAnomalyDimension() dim.Add("DataCenter", _aggSymbol); return dim; + Assert.NotNull(returnedRootCause); + Assert.Equal(1, (int)returnedRootCause.RootCause.Items.Count); + + foreach (KeyValuePair pair in returnedRootCause.RootCause.Items[0].Dimension) + { + Assert.Equal(expectedDim[pair.Key], pair.Value); + } + DeleteOutputPath(modelPath); } private static DateTime GetCurrentTimestamp() From bde4a53e185160a91459f2796c37700774938cab Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 16:50:23 +0800 Subject: [PATCH 18/36] update --- src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index 4a7cdaa086..ea9374cce0 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -414,6 +414,7 @@ private TimeSeriesPoint GetPointByDimension(Dictionary return p; } else + { { return null; } @@ -571,6 +572,8 @@ private double GetFinalScore(double surprise, double ep, double beta) a = (1 - Math.Pow(2, -surprise)); b = (1 - Math.Pow(2, -ep)); } + return beta * a + (1 - beta) * b; + } return beta * a + (1 - beta) * b; } @@ -668,6 +671,10 @@ private void UpdateDistribution(Dictionary distribution, List Double.IsNaN(val) ? 0 : Math.Log(val) / Math.Log(2); + if (Double.IsNaN(val)) + { + return 0; + } private static bool ContainsAll(Dictionary bigDictionary, Dictionary smallDictionary) { From a1ab905403d5498fb66559133d36e493b80e4fc0 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 16:52:30 +0800 Subject: [PATCH 19/36] merge --- src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index ea9374cce0..d6bc4e368c 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -362,6 +362,7 @@ private void GetRootCauseDirectionAndScore(Dictionary d else if (dst.Items.Count == 1) { TimeSeriesPoint rootCausePoint = GetPointByDimension(dimPointMapping, dst.Items[0].Dimension, pointTree, aggType, aggSymbol); + Point rootCausePoint = GetPointByDimension(dimPointMapping, dst.Items[0].Dimension, pointTree, aggType, aggSymbol); if (anomalyPoint != null && rootCausePoint != null) { Tuple scores = GetSurpriseAndExplanatoryScore(rootCausePoint, anomalyPoint); From 711dfca8f8012235e649a2f1f7def35ee883e281 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 16:53:51 +0800 Subject: [PATCH 20/36] merge --- src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs | 1 + src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index d6bc4e368c..7d258fc45a 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -742,6 +742,7 @@ public int CompareTo(object obj) return DimensionKey.CompareTo(other.DimensionKey); else throw new ArgumentException("Object is not a BestDimension"); + throw new ArgumentException("Object is not a BestDimension"); } } diff --git a/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs b/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs index a55a35060d..4bf1d4de22 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs @@ -46,6 +46,10 @@ public sealed class RootCauseLocalizationInput /// The string you defined as a aggregated symbol in the AnomalyDimension and point dimension. /// public Object AggregateSymbol { get; set; } + //When the anomaly incident occurs + //Point with the anomaly dimension must exist in the slice list at the anomaly timestamp, or the libary will not calculate the root cause + //A list of points at different timestamp. If the slices don't contain point data corresponding to the anomaly timestamp, the root cause localization alogorithm will not calculate the root cause as no information at the anomaly timestamp is provided. + //The aggregated symbol in the AnomalyDimension and point dimension should be consistent public RootCauseLocalizationInput(DateTime anomalyTimestamp, Dictionary anomalyDimension, List slices, AggregateType aggregateType, Object aggregateSymbol) { From e202a040c1560154f5a1b38fbf756e4219070876 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 16:58:23 +0800 Subject: [PATCH 21/36] merge --- .../time-series-root-cause-localization-dt.md | 47 --- .../time-series-root-cause-localization.md | 26 +- .../ExtensionsCatalog.cs | 8 +- .../RootCauseLocalization.cs | 294 ++++++++++++++++++ 4 files changed, 310 insertions(+), 65 deletions(-) delete mode 100644 docs/api-reference/time-series-root-cause-localization-dt.md create mode 100644 src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs diff --git a/docs/api-reference/time-series-root-cause-localization-dt.md b/docs/api-reference/time-series-root-cause-localization-dt.md deleted file mode 100644 index 16994b8c48..0000000000 --- a/docs/api-reference/time-series-root-cause-localization-dt.md +++ /dev/null @@ -1,47 +0,0 @@ -At Mircosoft, we develop a decision tree based root cause localization method which helps to find out the root causes for an anomaly incident incrementally. - -## Multi-Dimensional Root Cause Localization -It's a common case that one measure are collected with many dimensions (*e.g.*, Province, ISP) whose values are categorical(*e.g.*, Beijing or Shanghai for dimension Province). When a measure's value deviates from its expected value, this measure encounters anomalies. In such case, operators would like to localize the root cause dimension combinations rapidly and accurately. Multi-dimensional root cause localization is critical to troubleshoot and mitigate such case. - -## Algorithm - -The decision based root cause localization method is unsupervised, which means training step is no needed. It consists of the following major steps: -(1) Find best dimension which divides the anomaly and unanomaly data based on decision tree according to entropy gain and entropy gain ratio. -(2) Find the top anomaly points for the selected best dimension. - -### Decision Tree - -[Decision tree](https://en.wikipedia.org/wiki/Decision_tree) algorithm chooses the highest information gain to split or construct a decision tree.  We use it to choose the dimension which contributes the most to the anomaly. Following are some concepts used in decision tree. - -#### Information Entropy - -Information [entropy](https://en.wikipedia.org/wiki/Entropy_(information_theory)) is a measure of disorder or uncertainty. You can think of it as a measure of purity as well.The less the value , the more pure of data D. - -$$Ent(D) = - \sum_{k=1}^{|y|} p_k\log_2(p_k) $$ - -where $p_k$ represents the probability of an element in dataset. In our case, there are only two classed, the anomaly points and the normaly points. $|y|$ is the count of total anomalies. - -#### Information Gain -[Information gain](https://en.wikipedia.org/wiki/Information_gain_in_decision_trees) is a metric to measure the reduction of this disorder in our target class given additional information about it. Mathematically it can be written as: - -$$Gain(D, a) = Ent(D) - \sum_{v=1}^{|V|} \frac{|D^V|}{|D |} Ent(D^v) $$ - -Where $Ent(D^v)$ is the entropy of set points in D for which dimension $a$ is equal to $v$, $|D|$ is the total number of points in dataset $D$. $|D^V|$ is the total number of points in dataset $D$ for which dimension $a$ is equal to $v$. - -For all aggregated dimensions, we calculate the information for each dimension. The greater the reduction in this uncertainty, the more information is gained about D from dimension $a$. - -#### Entropy Gain Ratio - -Information gain is biased toward variables with large number of distinct values. A modification is [information gain ratio](https://en.wikipedia.org/wiki/Information_gain_ratio), which reduces its bias. - -$$Ratio(D, a) = \frac{Gain(D,a)} {IV(a)} $$ - -where intrinsic value(IV) is the entropy of split (with respect to dimension $a$ on focus). - -$$IV(a) = -\sum_{v=1}^V\frac{|D^v|} {|D|} \log_2 \frac{|D^v|} {|D|} $$ - -In out strategy, firstly, for all the aggration dimensions, we loop all the dimensions to find the dimension who's entropy gain is above mean entropy gain ration, then from the filtered dimensions, we select the dimension with highest entropy ratio as the best dimension. In the meanwhile, dimensions for which the anomaly value count is only one, we include it when calculation. - -> [!Note] -> 1. As our algorithm depends on the data you input, so if the input points is incorrect or incomplete, the calculated result will be unexpected. -> 2. Currently, the algorithm localize the root cause incrementally, which means at most one dimension with the values are detected. If you want to find out all the dimension that contributes to the anomaly, you can call this API recursively. diff --git a/docs/api-reference/time-series-root-cause-localization.md b/docs/api-reference/time-series-root-cause-localization.md index d3336b378a..16994b8c48 100644 --- a/docs/api-reference/time-series-root-cause-localization.md +++ b/docs/api-reference/time-series-root-cause-localization.md @@ -1,27 +1,25 @@ -At Microsoft, we have developed a decision tree based root cause localization method which helps to find out the root causes for an anomaly incident at a specific timestamp incrementally. +At Mircosoft, we develop a decision tree based root cause localization method which helps to find out the root causes for an anomaly incident incrementally. ## Multi-Dimensional Root Cause Localization -It's a common case that one measure is collected with many dimensions (*e.g.*, Province, ISP) whose values are categorical(*e.g.*, Beijing or Shanghai for dimension Province). When a measure's value deviates from its expected value, this measure encounters anomalies. In such case, users would like to localize the root cause dimension combinations rapidly and accurately. Multi-dimensional root cause localization is critical to troubleshoot and mitigate such case. +It's a common case that one measure are collected with many dimensions (*e.g.*, Province, ISP) whose values are categorical(*e.g.*, Beijing or Shanghai for dimension Province). When a measure's value deviates from its expected value, this measure encounters anomalies. In such case, operators would like to localize the root cause dimension combinations rapidly and accurately. Multi-dimensional root cause localization is critical to troubleshoot and mitigate such case. ## Algorithm -The decision tree based root cause localization method is unsupervised, which means training step is not needed. It consists of the following major steps: - -(1) Find the best dimension which divides the anomalous and regular data based on decision tree according to entropy gain and entropy gain ratio. - -(2) Find the top anomaly points which contribute the most to anomaly incident given the selected best dimension. +The decision based root cause localization method is unsupervised, which means training step is no needed. It consists of the following major steps: +(1) Find best dimension which divides the anomaly and unanomaly data based on decision tree according to entropy gain and entropy gain ratio. +(2) Find the top anomaly points for the selected best dimension. ### Decision Tree -The [Decision tree](https://en.wikipedia.org/wiki/Decision_tree) algorithm chooses the highest information gain to split or construct a decision tree.  We use it to choose the dimension which contributes the most to the anomaly. Below are some concepts used in decision trees. +[Decision tree](https://en.wikipedia.org/wiki/Decision_tree) algorithm chooses the highest information gain to split or construct a decision tree.  We use it to choose the dimension which contributes the most to the anomaly. Following are some concepts used in decision tree. #### Information Entropy -Information [entropy](https://en.wikipedia.org/wiki/Entropy_(information_theory)) is a measure of disorder or uncertainty. You can think of it as a measure of purity as well. The less the value , the more pure of data D. +Information [entropy](https://en.wikipedia.org/wiki/Entropy_(information_theory)) is a measure of disorder or uncertainty. You can think of it as a measure of purity as well.The less the value , the more pure of data D. $$Ent(D) = - \sum_{k=1}^{|y|} p_k\log_2(p_k) $$ -where $p_k$ represents the probability of an element in dataset. In our case, there are only two classes, the anomalous points and the regular points. $|y|$ is the count of total anomalies. +where $p_k$ represents the probability of an element in dataset. In our case, there are only two classed, the anomaly points and the normaly points. $|y|$ is the count of total anomalies. #### Information Gain [Information gain](https://en.wikipedia.org/wiki/Information_gain_in_decision_trees) is a metric to measure the reduction of this disorder in our target class given additional information about it. Mathematically it can be written as: @@ -30,7 +28,7 @@ $$Gain(D, a) = Ent(D) - \sum_{v=1}^{|V|} \frac{|D^V|}{|D |} Ent(D^v) $$ Where $Ent(D^v)$ is the entropy of set points in D for which dimension $a$ is equal to $v$, $|D|$ is the total number of points in dataset $D$. $|D^V|$ is the total number of points in dataset $D$ for which dimension $a$ is equal to $v$. -For all aggregated dimensions, we calculate the information for each dimension. The greater the reduction in this uncertainty, the more information is gained about $D$ from dimension $a$. +For all aggregated dimensions, we calculate the information for each dimension. The greater the reduction in this uncertainty, the more information is gained about D from dimension $a$. #### Entropy Gain Ratio @@ -38,12 +36,12 @@ Information gain is biased toward variables with large number of distinct values $$Ratio(D, a) = \frac{Gain(D,a)} {IV(a)} $$ -where intrinsic value($IV$) is the entropy of split (with respect to dimension $a$ on focus). +where intrinsic value(IV) is the entropy of split (with respect to dimension $a$ on focus). $$IV(a) = -\sum_{v=1}^V\frac{|D^v|} {|D|} \log_2 \frac{|D^v|} {|D|} $$ -In our strategy, firstly, for all the aggregated dimensions, we loop the dimension to find the dimension whose entropy gain is above mean entropy gain, then from the filtered dimensions, we select the dimension with highest entropy ratio as the best dimension. In the meanwhile, dimensions for which the anomaly value count is only one, we include it when calculation. +In out strategy, firstly, for all the aggration dimensions, we loop all the dimensions to find the dimension who's entropy gain is above mean entropy gain ration, then from the filtered dimensions, we select the dimension with highest entropy ratio as the best dimension. In the meanwhile, dimensions for which the anomaly value count is only one, we include it when calculation. > [!Note] > 1. As our algorithm depends on the data you input, so if the input points is incorrect or incomplete, the calculated result will be unexpected. -> 2. Currently, the algorithm localize the root cause incrementally, which means at most one dimension with the values are detected. If you want to find out all the dimensions that contribute to the anomaly, you can call this API recursively by updating the anomaly incident with the fixed dimension value. +> 2. Currently, the algorithm localize the root cause incrementally, which means at most one dimension with the values are detected. If you want to find out all the dimension that contributes to the anomaly, you can call this API recursively. diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index bc2b833218..96ca1060c7 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -226,7 +226,7 @@ private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocaliza } /// - /// Create , which localizes root causes using decision tree algorithm. + /// Create , which localizes root causes using decision tree algorithm. /// /// The transform's catalog. /// Name of the column resulting from the transformation of . @@ -237,12 +237,12 @@ private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocaliza /// /// /// /// /// - public static DTRootCauseLocalizationEstimator LocalizeRootCauseByDT(this TransformsCatalog catalog, string outputColumnName, string inputColumnName = null, double beta = 0.5) - => new DTRootCauseLocalizationEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, inputColumnName ?? outputColumnName, beta); + public static RootCauseLocalizationEstimator LocalizeRootCause(this TransformsCatalog catalog, string outputColumnName, string inputColumnName = null, double beta = 0.5) + => new RootCauseLocalizationEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, inputColumnName ?? outputColumnName, beta); /// /// Singular Spectrum Analysis (SSA) model for univariate time-series forecasting. diff --git a/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs b/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs new file mode 100644 index 0000000000..eea280f1e4 --- /dev/null +++ b/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs @@ -0,0 +1,294 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.ML; +using Microsoft.ML.CommandLine; +using Microsoft.ML.Data; +using Microsoft.ML.Internal.Utilities; +using Microsoft.ML.Runtime; +using Microsoft.ML.TimeSeries; +using Microsoft.ML.Transforms.TimeSeries; + +[assembly: LoadableClass(typeof(RootCauseLocalizationTransformer), null, typeof(SignatureLoadModel), + RootCauseLocalizationTransformer.UserName, RootCauseLocalizationTransformer.LoaderSignature)] + +[assembly: LoadableClass(typeof(IRowMapper), typeof(RootCauseLocalizationTransformer), null, typeof(SignatureLoadRowMapper), + RootCauseLocalizationTransformer.UserName, RootCauseLocalizationTransformer.LoaderSignature)] + +namespace Microsoft.ML.Transforms.TimeSeries +{ + /// + /// resulting from fitting an . + /// + public sealed class RootCauseLocalizationTransformer : OneToOneTransformerBase + { + internal const string Summary = "Localize root cause for anomaly."; + internal const string UserName = "Root Cause Localization Transform"; + internal const string LoaderSignature = "RootCauseTransform"; + + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "ROOTCAUSE", + verWrittenCur: 0x00010001, // Initial + verReadableCur: 0x00010001, + verWeCanReadBack: 0x00010001, + loaderSignature: LoaderSignature, + loaderAssemblyName: typeof(RootCauseLocalizationTransformer).Assembly.FullName); + } + + private const string RegistrationName = "RootCauseLocalization"; + + internal sealed class Column : OneToOneColumn + { + internal static Column Parse(string str) + { + var res = new Column(); + if (res.TryParse(str)) + return res; + return null; + } + + internal bool TryUnparse(StringBuilder sb) + { + Contracts.AssertValue(sb); + return TryUnparseCore(sb); + } + } + + internal class Options : TransformInputBase + { + [Argument(ArgumentType.Required, HelpText = "The name of the source column.", ShortName = "src", SortOrder = 1, Purpose = SpecialPurpose.ColumnName)] + public string Source; + + [Argument(ArgumentType.Required, HelpText = "The name of the output column.", SortOrder = 2)] + public string Output; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Weight for getting the score for the root cause item.", ShortName = "Beta", SortOrder = 2)] + public double Beta = RootCauseLocalizationEstimator.Defaults.Beta; + + } + + /// + /// The input and output column pairs passed to this . + /// + internal IReadOnlyCollection<(string outputColumnName, string inputColumnName)> Columns => ColumnPairs.AsReadOnly(); + + private readonly double _beta; + + /// + /// Localization root cause for multi-dimensional anomaly. + /// + /// The estimator's local . + /// Weight for generating score. + /// The name of the columns (first item of the tuple), and the name of the resulting output column (second item of the tuple). + + internal RootCauseLocalizationTransformer(IHostEnvironment env, double beta = RootCauseLocalizationEstimator.Defaults.Beta, params (string outputColumnName, string inputColumnName)[] columns) + : base(Contracts.CheckRef(env, nameof(env)).Register(RegistrationName), columns) + { + Host.CheckUserArg(beta >= 0 && beta <= 1, nameof(Options.Beta), "Must be in [0,1]"); + + _beta = beta; + } + + // Factory method for SignatureLoadModel. + private static RootCauseLocalizationTransformer Create(IHostEnvironment env, ModelLoadContext ctx) + { + Contracts.CheckValue(env, nameof(env)); + var host = env.Register(RegistrationName); + host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(GetVersionInfo()); + return new RootCauseLocalizationTransformer(host, ctx); + } + + private RootCauseLocalizationTransformer(IHost host, ModelLoadContext ctx) + : base(host, ctx) + { + // *** Binary format *** + // + // double: beta + _beta = ctx.Reader.ReadDouble(); + Host.CheckDecode(_beta >= 0 && _beta <= 1); + } + + // Factory method for SignatureLoadDataTransform. + private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) + => Create(env, ctx).MakeDataTransform(input); + + // Factory method for SignatureLoadRowMapper. + private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema) + => Create(env, ctx).MakeRowMapper(inputSchema); + + private protected override void SaveModel(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); + + // *** Binary format *** + // + base.SaveColumns(ctx); + // double: beta + ctx.Writer.Write(_beta); + } + + private protected override IRowMapper MakeRowMapper(DataViewSchema schema) => new Mapper(this, schema); + + private protected override void CheckInputColumn(DataViewSchema inputSchema, int col, int srcCol) + { + if (!(inputSchema[srcCol].Type is RootCauseLocalizationInputDataViewType)) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", ColumnPairs[col].inputColumnName, "RootCauseLocalizationInputDataViewType", inputSchema[srcCol].Type.ToString()); + } + + private sealed class Mapper : OneToOneMapperBase + { + private readonly RootCauseLocalizationTransformer _parent; + + public Mapper(RootCauseLocalizationTransformer parent, DataViewSchema inputSchema) + : base(parent.Host.Register(nameof(Mapper)), parent, inputSchema) + { + _parent = parent; + } + + protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore() + { + var result = new DataViewSchema.DetachedColumn[_parent.ColumnPairs.Length]; + for (int i = 0; i < _parent.ColumnPairs.Length; i++) + { + InputSchema.TryGetColumnIndex(_parent.ColumnPairs[i].inputColumnName, out int colIndex); + Host.Assert(colIndex >= 0); + + DataViewType type; + type = new RootCauseDataViewType(); + + result[i] = new DataViewSchema.DetachedColumn(_parent.ColumnPairs[i].outputColumnName, type, null); + } + return result; + } + + protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func activeOutput, out Action disposer) + { + Contracts.AssertValue(input); + Contracts.Assert(0 <= iinfo && iinfo < _parent.ColumnPairs.Length); + + var src = default(RootCauseLocalizationInput); + var getSrc = input.GetGetter(input.Schema[ColMapNewToOld[iinfo]]); + + disposer = null; + + ValueGetter del = + (ref RootCause dst) => + { + getSrc(ref src); + if (src == null) + return; + + CheckRootCauseInput(src, Host); + + LocalizeRootCauses(src, ref dst); + }; + + return del; + } + + private void CheckRootCauseInput(RootCauseLocalizationInput src, IHost host) + { + if (src.Slices.Count < 1) + { + throw host.Except($"Length of Slices must be larger than 0"); + } + + bool containsAnomalyTimestamp = false; + foreach (MetricSlice slice in src.Slices) + { + if (slice.TimeStamp.Equals(src.AnomalyTimestamp)) + { + containsAnomalyTimestamp = true; + } + } + if (!containsAnomalyTimestamp) + { + throw host.Except($"Has no points in the given anomaly timestamp"); + } + } + + private void LocalizeRootCauses(RootCauseLocalizationInput src, ref RootCause dst) + { + RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, _parent._beta); + dst = analyzer.Analyze(); + } + } + } + + /// + /// for the . + /// + /// + /// | + /// | Output column data type | | + /// | Exportable to ONNX | No | + /// + /// [!include[io](~/../docs/samples/docs/api-reference/time-series-root-cause-localization.md)] + /// + /// The resulting creates a new column, named as specified in the output column name parameters, and + /// localize the root causes which contribute most to the anomaly. + /// Check the See Also section for links to usage examples. + /// ]]> + /// + /// + /// + public sealed class RootCauseLocalizationEstimator : TrivialEstimator + { + internal static class Defaults + { + public const double Beta = 0.5; + } + + /// + /// Localize root cause. + /// + /// The estimator's local . + /// Name of output column to run the root cause localization. + /// Name of input column to run the root cause localization. + /// The weight for generating score in output result. + [BestFriend] + internal RootCauseLocalizationEstimator(IHostEnvironment env, string outputColumnName, string inputColumnName, double beta = Defaults.Beta) + : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(RootCauseLocalizationEstimator)), new RootCauseLocalizationTransformer(env, beta, new[] { (outputColumnName, inputColumnName ?? outputColumnName) })) + { + } + + /// + /// Returns the of the schema which will be produced by the transformer. + /// Used for schema propagation and verification in a pipeline. + /// + public override SchemaShape GetOutputSchema(SchemaShape inputSchema) + { + Host.CheckValue(inputSchema, nameof(inputSchema)); + var result = inputSchema.ToDictionary(x => x.Name); + foreach (var colInfo in Transformer.Columns) + { + if (!inputSchema.TryFindColumn(colInfo.inputColumnName, out var col)) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colInfo.inputColumnName); + if (!(col.ItemType is RootCauseLocalizationInputDataViewType) || col.Kind != SchemaShape.Column.VectorKind.Scalar) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colInfo.inputColumnName, new RootCauseLocalizationInputDataViewType().ToString(), col.GetTypeString()); + + result[colInfo.outputColumnName] = new SchemaShape.Column(colInfo.outputColumnName, SchemaShape.Column.VectorKind.Scalar, new RootCauseDataViewType(), false); + } + + return new SchemaShape(result.Values); + } + } +} From 1a2d56913be79f75757454b583d34fb2d3eff9c2 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Mon, 27 Apr 2020 10:27:45 +0800 Subject: [PATCH 22/36] fix type --- src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs | 2 ++ src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index 7d258fc45a..b660a02ca1 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -573,6 +573,7 @@ private double GetFinalScore(double surprise, double ep, double beta) a = (1 - Math.Pow(2, -surprise)); b = (1 - Math.Pow(2, -ep)); } + return beta * a + (1 - beta) * b; } @@ -677,6 +678,7 @@ private void UpdateDistribution(Dictionary distribution, List bigDictionary, Dictionary smallDictionary) { foreach (var item in smallDictionary) diff --git a/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs b/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs index eea280f1e4..d90ce65d96 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs @@ -243,7 +243,7 @@ private void LocalizeRootCauses(RootCauseLocalizationInput src, ref RootCause ds /// /// [!include[io](~/../docs/samples/docs/api-reference/time-series-root-cause-localization.md)] /// - /// The resulting creates a new column, named as specified in the output column name parameters, and + /// The resulting creates a new column, named as specified in the output column name parameters, and /// localize the root causes which contribute most to the anomaly. /// Check the See Also section for links to usage examples. /// ]]> From b059b535ebbc75b0ff121d866ed1d855aced092b Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Tue, 28 Apr 2020 14:25:50 +0800 Subject: [PATCH 23/36] update model signature --- src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs b/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs index d90ce65d96..bfbc4af904 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs @@ -34,7 +34,7 @@ public sealed class RootCauseLocalizationTransformer : OneToOneTransformerBase private static VersionInfo GetVersionInfo() { return new VersionInfo( - modelSignature: "ROOTCAUSE", + modelSignature: "ROOTCAUS", verWrittenCur: 0x00010001, // Initial verReadableCur: 0x00010001, verWeCanReadBack: 0x00010001, From fa834fd082dabcd67b4265f422780335266800c4 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Thu, 7 May 2020 15:47:10 +0800 Subject: [PATCH 24/36] update rca interface by removing transformer --- .../time-series-root-cause-localization.md | 22 +- .../ExtensionsCatalog.cs | 51 ++- .../RootCauseAnalyzer.cs | 1 + .../RootCauseLocalization.cs | 294 ------------------ .../TimeSeriesDirectApi.cs | 69 +--- 5 files changed, 67 insertions(+), 370 deletions(-) delete mode 100644 src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs diff --git a/docs/api-reference/time-series-root-cause-localization.md b/docs/api-reference/time-series-root-cause-localization.md index 16994b8c48..3eecc838e2 100644 --- a/docs/api-reference/time-series-root-cause-localization.md +++ b/docs/api-reference/time-series-root-cause-localization.md @@ -1,13 +1,15 @@ -At Mircosoft, we develop a decision tree based root cause localization method which helps to find out the root causes for an anomaly incident incrementally. +At Mircosoft, we develop a decision tree based root cause localization method which helps to find out the root causes for an anomaly incident at a specific timestamp incrementally. ## Multi-Dimensional Root Cause Localization -It's a common case that one measure are collected with many dimensions (*e.g.*, Province, ISP) whose values are categorical(*e.g.*, Beijing or Shanghai for dimension Province). When a measure's value deviates from its expected value, this measure encounters anomalies. In such case, operators would like to localize the root cause dimension combinations rapidly and accurately. Multi-dimensional root cause localization is critical to troubleshoot and mitigate such case. +It's a common case that one measure is collected with many dimensions (*e.g.*, Province, ISP) whose values are categorical(*e.g.*, Beijing or Shanghai for dimension Province). When a measure's value deviates from its expected value, this measure encounters anomalies. In such case, operators would like to localize the root cause dimension combinations rapidly and accurately. Multi-dimensional root cause localization is critical to troubleshoot and mitigate such case. ## Algorithm -The decision based root cause localization method is unsupervised, which means training step is no needed. It consists of the following major steps: -(1) Find best dimension which divides the anomaly and unanomaly data based on decision tree according to entropy gain and entropy gain ratio. -(2) Find the top anomaly points for the selected best dimension. +The decision tree based root cause localization method is unsupervised, which means training step is not needed. It consists of the following major steps: + +(1) Find the best dimension which divides the anomalous and regular data based on decision tree according to entropy gain and entropy gain ratio. + +(2) Find the top anomaly points which contribute the most to anomaly incident given the selected best dimension. ### Decision Tree @@ -15,11 +17,11 @@ The decision based root cause localization method is unsupervised, which means t #### Information Entropy -Information [entropy](https://en.wikipedia.org/wiki/Entropy_(information_theory)) is a measure of disorder or uncertainty. You can think of it as a measure of purity as well.The less the value , the more pure of data D. +Information [entropy](https://en.wikipedia.org/wiki/Entropy_(information_theory)) is a measure of disorder or uncertainty. You can think of it as a measure of purity as well. The less the value , the more pure of data D. $$Ent(D) = - \sum_{k=1}^{|y|} p_k\log_2(p_k) $$ -where $p_k$ represents the probability of an element in dataset. In our case, there are only two classed, the anomaly points and the normaly points. $|y|$ is the count of total anomalies. +where $p_k$ represents the probability of an element in dataset. In our case, there are only two classes, the anomalous points and the regular points. $|y|$ is the count of total anomalies. #### Information Gain [Information gain](https://en.wikipedia.org/wiki/Information_gain_in_decision_trees) is a metric to measure the reduction of this disorder in our target class given additional information about it. Mathematically it can be written as: @@ -36,12 +38,12 @@ Information gain is biased toward variables with large number of distinct values $$Ratio(D, a) = \frac{Gain(D,a)} {IV(a)} $$ -where intrinsic value(IV) is the entropy of split (with respect to dimension $a$ on focus). +where intrinsic value($IV$) is the entropy of split (with respect to dimension $a$ on focus). $$IV(a) = -\sum_{v=1}^V\frac{|D^v|} {|D|} \log_2 \frac{|D^v|} {|D|} $$ -In out strategy, firstly, for all the aggration dimensions, we loop all the dimensions to find the dimension who's entropy gain is above mean entropy gain ration, then from the filtered dimensions, we select the dimension with highest entropy ratio as the best dimension. In the meanwhile, dimensions for which the anomaly value count is only one, we include it when calculation. +In our strategy, firstly, for all the aggregated dimensions, we loop the dimension to find the dimension whose entropy gain is above mean entropy gain, then from the filtered dimensions, we select the dimension with highest entropy ratio as the best dimension. In the meanwhile, dimensions for which the anomaly value count is only one, we include it when calculation. > [!Note] > 1. As our algorithm depends on the data you input, so if the input points is incorrect or incomplete, the calculated result will be unexpected. -> 2. Currently, the algorithm localize the root cause incrementally, which means at most one dimension with the values are detected. If you want to find out all the dimension that contributes to the anomaly, you can call this API recursively. +> 2. Currently, the algorithm localize the root cause incrementally, which means at most one dimension with the values are detected. If you want to find out all the dimensions that contribute to the anomaly, you can call this API recursively by updating the anomaly incident with the fixed dimension value. diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 96ca1060c7..1b7876db7e 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; +using System.Reflection; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TimeSeries; @@ -226,14 +228,11 @@ private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocaliza } /// - /// Create , which localizes root causes using decision tree algorithm. + /// Create , which localizes root causes using decision tree algorithm. /// - /// The transform's catalog. - /// Name of the column resulting from the transformation of . - /// The column data is an instance of . - /// Name of the input column. - /// The column data is an instance of . - /// The weight parameter in score. The range of the parameter should be in [0,1]. + /// The anomaly detection catalog. + /// Root cause's input. The data is an instance of . + /// Beta is a weight parameter for user to choose. It is used when score is calculated for each root cause item. The range of beta should be in [0,1]. For a larger beta, root cause point which has a large difference between value and expected value will get a high score. On the contrary, for a small beta, root cause items which has a high relative change will get a high score. /// /// /// /// /// - public static RootCauseLocalizationEstimator LocalizeRootCause(this TransformsCatalog catalog, string outputColumnName, string inputColumnName = null, double beta = 0.5) - => new RootCauseLocalizationEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnName, inputColumnName ?? outputColumnName, beta); + public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, RootCauseLocalizationInput src, double beta = 0.5) + { + //check the root cause input + CheckRootCauseInput(src); + + //check beta + if (beta < 0 || beta > 1) { + throw new ArgumentException("Beta must be in [0,1]"); + } + + //find out the root cause + RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, beta); + RootCause dst = analyzer.Analyze(); + return dst; + } + + private static void CheckRootCauseInput(RootCauseLocalizationInput src) + { + if (src.Slices.Count < 1) + { + throw new ArgumentException("Length of Slices must be larger than 0"); + } + + bool containsAnomalyTimestamp = false; + foreach (MetricSlice slice in src.Slices) + { + if (slice.TimeStamp.Equals(src.AnomalyTimestamp)) + { + containsAnomalyTimestamp = true; + } + } + if (!containsAnomalyTimestamp) + { + throw new ArgumentException("Has no points in the given anomaly timestamp"); + } + } /// /// Singular Spectrum Analysis (SSA) model for univariate time-series forecasting. diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index b660a02ca1..9dc0c42af5 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -357,6 +357,7 @@ private void GetRootCauseDirectionAndScore(Dictionary d { dst.Items[i].Score = GetFinalScore(scoreList[i].Surprise, Math.Abs(scoreList[i].ExplanatoryScore), beta); } + } } } else if (dst.Items.Count == 1) diff --git a/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs b/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs deleted file mode 100644 index bfbc4af904..0000000000 --- a/src/Microsoft.ML.TimeSeries/RootCauseLocalization.cs +++ /dev/null @@ -1,294 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Microsoft.ML; -using Microsoft.ML.CommandLine; -using Microsoft.ML.Data; -using Microsoft.ML.Internal.Utilities; -using Microsoft.ML.Runtime; -using Microsoft.ML.TimeSeries; -using Microsoft.ML.Transforms.TimeSeries; - -[assembly: LoadableClass(typeof(RootCauseLocalizationTransformer), null, typeof(SignatureLoadModel), - RootCauseLocalizationTransformer.UserName, RootCauseLocalizationTransformer.LoaderSignature)] - -[assembly: LoadableClass(typeof(IRowMapper), typeof(RootCauseLocalizationTransformer), null, typeof(SignatureLoadRowMapper), - RootCauseLocalizationTransformer.UserName, RootCauseLocalizationTransformer.LoaderSignature)] - -namespace Microsoft.ML.Transforms.TimeSeries -{ - /// - /// resulting from fitting an . - /// - public sealed class RootCauseLocalizationTransformer : OneToOneTransformerBase - { - internal const string Summary = "Localize root cause for anomaly."; - internal const string UserName = "Root Cause Localization Transform"; - internal const string LoaderSignature = "RootCauseTransform"; - - private static VersionInfo GetVersionInfo() - { - return new VersionInfo( - modelSignature: "ROOTCAUS", - verWrittenCur: 0x00010001, // Initial - verReadableCur: 0x00010001, - verWeCanReadBack: 0x00010001, - loaderSignature: LoaderSignature, - loaderAssemblyName: typeof(RootCauseLocalizationTransformer).Assembly.FullName); - } - - private const string RegistrationName = "RootCauseLocalization"; - - internal sealed class Column : OneToOneColumn - { - internal static Column Parse(string str) - { - var res = new Column(); - if (res.TryParse(str)) - return res; - return null; - } - - internal bool TryUnparse(StringBuilder sb) - { - Contracts.AssertValue(sb); - return TryUnparseCore(sb); - } - } - - internal class Options : TransformInputBase - { - [Argument(ArgumentType.Required, HelpText = "The name of the source column.", ShortName = "src", SortOrder = 1, Purpose = SpecialPurpose.ColumnName)] - public string Source; - - [Argument(ArgumentType.Required, HelpText = "The name of the output column.", SortOrder = 2)] - public string Output; - - [Argument(ArgumentType.AtMostOnce, HelpText = "Weight for getting the score for the root cause item.", ShortName = "Beta", SortOrder = 2)] - public double Beta = RootCauseLocalizationEstimator.Defaults.Beta; - - } - - /// - /// The input and output column pairs passed to this . - /// - internal IReadOnlyCollection<(string outputColumnName, string inputColumnName)> Columns => ColumnPairs.AsReadOnly(); - - private readonly double _beta; - - /// - /// Localization root cause for multi-dimensional anomaly. - /// - /// The estimator's local . - /// Weight for generating score. - /// The name of the columns (first item of the tuple), and the name of the resulting output column (second item of the tuple). - - internal RootCauseLocalizationTransformer(IHostEnvironment env, double beta = RootCauseLocalizationEstimator.Defaults.Beta, params (string outputColumnName, string inputColumnName)[] columns) - : base(Contracts.CheckRef(env, nameof(env)).Register(RegistrationName), columns) - { - Host.CheckUserArg(beta >= 0 && beta <= 1, nameof(Options.Beta), "Must be in [0,1]"); - - _beta = beta; - } - - // Factory method for SignatureLoadModel. - private static RootCauseLocalizationTransformer Create(IHostEnvironment env, ModelLoadContext ctx) - { - Contracts.CheckValue(env, nameof(env)); - var host = env.Register(RegistrationName); - host.CheckValue(ctx, nameof(ctx)); - ctx.CheckAtModel(GetVersionInfo()); - return new RootCauseLocalizationTransformer(host, ctx); - } - - private RootCauseLocalizationTransformer(IHost host, ModelLoadContext ctx) - : base(host, ctx) - { - // *** Binary format *** - // - // double: beta - _beta = ctx.Reader.ReadDouble(); - Host.CheckDecode(_beta >= 0 && _beta <= 1); - } - - // Factory method for SignatureLoadDataTransform. - private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) - => Create(env, ctx).MakeDataTransform(input); - - // Factory method for SignatureLoadRowMapper. - private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema) - => Create(env, ctx).MakeRowMapper(inputSchema); - - private protected override void SaveModel(ModelSaveContext ctx) - { - Host.CheckValue(ctx, nameof(ctx)); - - ctx.CheckAtModel(); - ctx.SetVersionInfo(GetVersionInfo()); - - // *** Binary format *** - // - base.SaveColumns(ctx); - // double: beta - ctx.Writer.Write(_beta); - } - - private protected override IRowMapper MakeRowMapper(DataViewSchema schema) => new Mapper(this, schema); - - private protected override void CheckInputColumn(DataViewSchema inputSchema, int col, int srcCol) - { - if (!(inputSchema[srcCol].Type is RootCauseLocalizationInputDataViewType)) - throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", ColumnPairs[col].inputColumnName, "RootCauseLocalizationInputDataViewType", inputSchema[srcCol].Type.ToString()); - } - - private sealed class Mapper : OneToOneMapperBase - { - private readonly RootCauseLocalizationTransformer _parent; - - public Mapper(RootCauseLocalizationTransformer parent, DataViewSchema inputSchema) - : base(parent.Host.Register(nameof(Mapper)), parent, inputSchema) - { - _parent = parent; - } - - protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore() - { - var result = new DataViewSchema.DetachedColumn[_parent.ColumnPairs.Length]; - for (int i = 0; i < _parent.ColumnPairs.Length; i++) - { - InputSchema.TryGetColumnIndex(_parent.ColumnPairs[i].inputColumnName, out int colIndex); - Host.Assert(colIndex >= 0); - - DataViewType type; - type = new RootCauseDataViewType(); - - result[i] = new DataViewSchema.DetachedColumn(_parent.ColumnPairs[i].outputColumnName, type, null); - } - return result; - } - - protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func activeOutput, out Action disposer) - { - Contracts.AssertValue(input); - Contracts.Assert(0 <= iinfo && iinfo < _parent.ColumnPairs.Length); - - var src = default(RootCauseLocalizationInput); - var getSrc = input.GetGetter(input.Schema[ColMapNewToOld[iinfo]]); - - disposer = null; - - ValueGetter del = - (ref RootCause dst) => - { - getSrc(ref src); - if (src == null) - return; - - CheckRootCauseInput(src, Host); - - LocalizeRootCauses(src, ref dst); - }; - - return del; - } - - private void CheckRootCauseInput(RootCauseLocalizationInput src, IHost host) - { - if (src.Slices.Count < 1) - { - throw host.Except($"Length of Slices must be larger than 0"); - } - - bool containsAnomalyTimestamp = false; - foreach (MetricSlice slice in src.Slices) - { - if (slice.TimeStamp.Equals(src.AnomalyTimestamp)) - { - containsAnomalyTimestamp = true; - } - } - if (!containsAnomalyTimestamp) - { - throw host.Except($"Has no points in the given anomaly timestamp"); - } - } - - private void LocalizeRootCauses(RootCauseLocalizationInput src, ref RootCause dst) - { - RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, _parent._beta); - dst = analyzer.Analyze(); - } - } - } - - /// - /// for the . - /// - /// - /// | - /// | Output column data type | | - /// | Exportable to ONNX | No | - /// - /// [!include[io](~/../docs/samples/docs/api-reference/time-series-root-cause-localization.md)] - /// - /// The resulting creates a new column, named as specified in the output column name parameters, and - /// localize the root causes which contribute most to the anomaly. - /// Check the See Also section for links to usage examples. - /// ]]> - /// - /// - /// - public sealed class RootCauseLocalizationEstimator : TrivialEstimator - { - internal static class Defaults - { - public const double Beta = 0.5; - } - - /// - /// Localize root cause. - /// - /// The estimator's local . - /// Name of output column to run the root cause localization. - /// Name of input column to run the root cause localization. - /// The weight for generating score in output result. - [BestFriend] - internal RootCauseLocalizationEstimator(IHostEnvironment env, string outputColumnName, string inputColumnName, double beta = Defaults.Beta) - : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(RootCauseLocalizationEstimator)), new RootCauseLocalizationTransformer(env, beta, new[] { (outputColumnName, inputColumnName ?? outputColumnName) })) - { - } - - /// - /// Returns the of the schema which will be produced by the transformer. - /// Used for schema propagation and verification in a pipeline. - /// - public override SchemaShape GetOutputSchema(SchemaShape inputSchema) - { - Host.CheckValue(inputSchema, nameof(inputSchema)); - var result = inputSchema.ToDictionary(x => x.Name); - foreach (var colInfo in Transformer.Columns) - { - if (!inputSchema.TryFindColumn(colInfo.inputColumnName, out var col)) - throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colInfo.inputColumnName); - if (!(col.ItemType is RootCauseLocalizationInputDataViewType) || col.Kind != SchemaShape.Column.VectorKind.Scalar) - throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colInfo.inputColumnName, new RootCauseLocalizationInputDataViewType().ToString(), col.GetTypeString()); - - result[colInfo.outputColumnName] = new SchemaShape.Column(colInfo.outputColumnName, SchemaShape.Column.VectorKind.Scalar, new RootCauseDataViewType(), false); - } - - return new SchemaShape(result.Values); - } - } -} diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index 188cddd05c..c5da84441d 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -102,32 +102,6 @@ private sealed class SrCnnAnomalyDetection private static Object _rootCauseAggSymbol = "##SUM##"; - private class RootCauseLocalizationData - { - [RootCauseLocalizationInputType] - public RootCauseLocalizationInput Input { get; set; } - - public RootCauseLocalizationData() - { - Input = null; - } - - public RootCauseLocalizationData(DateTime anomalyTimestamp, Dictionary anomalyDimensions, List slices, AggregateType aggregateteType, string aggregateSymbol) - { - Input = new RootCauseLocalizationInput(anomalyTimestamp, anomalyDimensions, slices, aggregateteType, aggregateSymbol); - } - } - - private class RootCauseLocalizationTransformedData - { - [RootCauseType()] - public RootCause RootCause { get; set; } - - public RootCauseLocalizationTransformedData() - { - RootCause = null; - } - } [Fact] public void ChangeDetection() @@ -629,7 +603,6 @@ public void TestSrCnnBatchAnomalyDetector( { data.Add(new TimeSeriesDataDouble { Value = 5 }); } - // Convert data to IDataView. dataView = ml.Data.LoadFromEnumerable(data); } @@ -639,12 +612,10 @@ public void TestSrCnnBatchAnomalyDetector( // Do batch anomaly detection var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName, threshold: 0.35, batchSize: batchSize, sensitivity: 90.0, mode); - // Getting the data of the newly created column as an IEnumerable of // SrCnnAnomalyDetection. var predictionColumn = ml.Data.CreateEnumerable( outputDataView, reuseRowObject: false); - int k = 0; foreach (var prediction in predictionColumn) switch (mode) @@ -704,7 +675,6 @@ public void RootCauseLocalization() Assert.Equal(expectedDim[pair.Key], pair.Value); } var newRootCauseInput = new RootCauseLocalizationData(timeStamp, GetAnomalyDimension(), new List() { new MetricSlice(timeStamp, GetRootCauseLocalizationPoints()) }, AggregateType.Sum, _aggSymbol); - Dictionary expectedDim = new Dictionary(); expectedDim.Add("Country","UK"); expectedDim.Add("DeviceType",_aggSymbol); @@ -773,10 +743,8 @@ private static Dictionary GetRootCauseAnomalyDimension() private static DateTime GetRootCauseTimestamp() { return new DateTime(2020, 3, 23, 0, 0, 0); - return points; } - private static Dictionary GetAnomalyDimension() { Dictionary dim = new Dictionary(); @@ -785,32 +753,28 @@ private static Dictionary GetAnomalyDimension() dim.Add("DataCenter", _aggSymbol); return dim; } - private static DateTime GetCurrentTimestamp() { return new DateTime(); Dictionary expectedDim = new Dictionary(); expectedDim.Add("Country", "UK"); - expectedDim.Add("DeviceType", _aggSymbol); + expectedDim.Add("DeviceType", _rootCauseAggSymbol); expectedDim.Add("DataCenter", "DC1"); - foreach (KeyValuePair pair in transformedRootCause.RootCause.Items[0].Dimension) + foreach (KeyValuePair pair in rootCause.Items[0].Dimension) { Assert.Equal(expectedDim[pair.Key], pair.Value); } } - private static List GetRootCauseLocalizationPoints() { List points = new List(); - Dictionary dic1 = new Dictionary(); dic1.Add("Country", "UK"); dic1.Add("DeviceType", "Laptop"); dic1.Add("DataCenter", "DC1"); points.Add(new Point(200, 100, true, dic1)); - Dictionary dic2 = new Dictionary(); dic2.Add("Country", "UK"); dic2.Add("DeviceType", "Mobile"); @@ -822,7 +786,6 @@ private static List GetRootCauseLocalizationPoints() dic3.Add("DeviceType", _aggSymbol); dic3.Add("DataCenter", "DC1"); points.Add(new Point(1200, 200, true, dic3)); - Dictionary dic4 = new Dictionary(); dic4.Add("Country", "UK"); dic4.Add("DeviceType", "Laptop"); @@ -870,14 +833,6 @@ private static Dictionary GetAnomalyDimension() dim.Add("DataCenter", _aggSymbol); return dim; - Assert.NotNull(returnedRootCause); - Assert.Equal(1, (int)returnedRootCause.RootCause.Items.Count); - - foreach (KeyValuePair pair in returnedRootCause.RootCause.Items[0].Dimension) - { - Assert.Equal(expectedDim[pair.Key], pair.Value); - } - DeleteOutputPath(modelPath); } private static DateTime GetCurrentTimestamp() @@ -903,7 +858,7 @@ private static List GetRootCauseLocalizationPoints() Dictionary dic3 = new Dictionary(); dic3.Add("Country", "UK"); - dic3.Add("DeviceType", _aggSymbol); + dic3.Add("DeviceType", _rootCauseAggSymbol); dic3.Add("DataCenter", "DC1"); points.Add(new Point(1200, 200, true, dic3)); @@ -921,42 +876,42 @@ private static List GetRootCauseLocalizationPoints() Dictionary dic6 = new Dictionary(); dic6.Add("Country", "UK"); - dic6.Add("DeviceType", _aggSymbol); + dic6.Add("DeviceType", _rootCauseAggSymbol); dic6.Add("DataCenter", "DC2"); points.Add(new Point(300, 300, false, dic6)); Dictionary dic7 = new Dictionary(); dic7.Add("Country", "UK"); - dic7.Add("DeviceType", _aggSymbol); - dic7.Add("DataCenter", _aggSymbol); + dic7.Add("DeviceType", _rootCauseAggSymbol); + dic7.Add("DataCenter", _rootCauseAggSymbol); points.Add(new Point(1500, 500, true, dic7)); Dictionary dic8 = new Dictionary(); dic8.Add("Country", "UK"); dic8.Add("DeviceType", "Laptop"); - dic8.Add("DataCenter", _aggSymbol); + dic8.Add("DataCenter", _rootCauseAggSymbol); points.Add(new Point(300, 200, true, dic8)); Dictionary dic9 = new Dictionary(); dic9.Add("Country", "UK"); dic9.Add("DeviceType", "Mobile"); - dic9.Add("DataCenter", _aggSymbol); + dic9.Add("DataCenter", _rootCauseAggSymbol); points.Add(new Point(1200, 300, true, dic9)); return points; } - private static Dictionary GetAnomalyDimension() + private static Dictionary GetRootCauseAnomalyDimension() { Dictionary dim = new Dictionary(); dim.Add("Country", "UK"); - dim.Add("DeviceType", _aggSymbol); - dim.Add("DataCenter", _aggSymbol); + dim.Add("DeviceType", _rootCauseAggSymbol); + dim.Add("DataCenter", _rootCauseAggSymbol); return dim; } - private static DateTime GetCurrentTimestamp() + private static DateTime GetRootCauseTimestamp() { return new DateTime(2020, 3, 23, 0, 0, 0); } From 2b2942664720f6d229ce40f6dd6488031313ae74 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Fri, 8 May 2020 21:12:21 +0800 Subject: [PATCH 25/36] update --- src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs | 15 +++++++-------- src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs | 5 ----- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 1b7876db7e..011a58e4e2 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -242,12 +242,14 @@ private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocaliza /// public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, RootCauseLocalizationInput src, double beta = 0.5) { + IHostEnvironment host = CatalogUtils.GetEnvironment(catalog); + //check the root cause input - CheckRootCauseInput(src); + CheckRootCauseInput(host, src); //check beta if (beta < 0 || beta > 1) { - throw new ArgumentException("Beta must be in [0,1]"); + host.CheckUserArg(beta >= 0 && beta <= 1, nameof(beta), "Must be in [0,1]"); } //find out the root cause @@ -256,11 +258,11 @@ public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, return dst; } - private static void CheckRootCauseInput(RootCauseLocalizationInput src) + private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocalizationInput src) { if (src.Slices.Count < 1) { - throw new ArgumentException("Length of Slices must be larger than 0"); + host.CheckUserArg(src.Slices.Count > 1 , nameof(src.Slices), "Must has more than one item"); } bool containsAnomalyTimestamp = false; @@ -271,10 +273,7 @@ private static void CheckRootCauseInput(RootCauseLocalizationInput src) containsAnomalyTimestamp = true; } } - if (!containsAnomalyTimestamp) - { - throw new ArgumentException("Has no points in the given anomaly timestamp"); - } + host.CheckUserArg(containsAnomalyTimestamp, nameof(src.Slices), "Has no points in the given anomaly timestamp"); } /// diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index 9dc0c42af5..b7112b1f8d 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -674,11 +674,6 @@ private void UpdateDistribution(Dictionary distribution, List Double.IsNaN(val) ? 0 : Math.Log(val) / Math.Log(2); - if (Double.IsNaN(val)) - { - return 0; - } - private static bool ContainsAll(Dictionary bigDictionary, Dictionary smallDictionary) { From 0860e0127a65ee8abed1fb05231758e7a0102943 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Sat, 9 May 2020 10:43:18 +0800 Subject: [PATCH 26/36] update --- src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 011a58e4e2..37b9bce044 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -248,22 +248,17 @@ public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, CheckRootCauseInput(host, src); //check beta - if (beta < 0 || beta > 1) { - host.CheckUserArg(beta >= 0 && beta <= 1, nameof(beta), "Must be in [0,1]"); - } + host.CheckUserArg(beta >= 0 && beta <= 1, nameof(beta), "Must be in [0,1]"); //find out the root cause RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, beta); - RootCause dst = analyzer.Analyze(); + RootCause dst = analyzer.Analyze(); return dst; } private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocalizationInput src) { - if (src.Slices.Count < 1) - { - host.CheckUserArg(src.Slices.Count > 1 , nameof(src.Slices), "Must has more than one item"); - } + host.CheckUserArg(src.Slices.Count >= 1, nameof(src.Slices), "Must has more than one item"); bool containsAnomalyTimestamp = false; foreach (MetricSlice slice in src.Slices) From 6cf15c8cb8d1516aa82eb1b8bd2b9ce8ce4c79de Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 17:33:52 +0800 Subject: [PATCH 27/36] merge --- .../ExtensionsCatalog.cs | 44 ---- .../RootCauseAnalyzer.cs | 9 +- .../RootCauseLocalizationType.cs | 35 ++- .../TimeSeriesDirectApi.cs | 210 +++--------------- 4 files changed, 46 insertions(+), 252 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 37b9bce044..7cba0ee711 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -227,50 +227,6 @@ private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocaliza host.CheckUserArg(containsAnomalyTimestamp, nameof(src.Slices), "Has no points in the given anomaly timestamp"); } - /// - /// Create , which localizes root causes using decision tree algorithm. - /// - /// The anomaly detection catalog. - /// Root cause's input. The data is an instance of . - /// Beta is a weight parameter for user to choose. It is used when score is calculated for each root cause item. The range of beta should be in [0,1]. For a larger beta, root cause point which has a large difference between value and expected value will get a high score. On the contrary, for a small beta, root cause items which has a high relative change will get a high score. - /// - /// - /// - /// - /// - public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, RootCauseLocalizationInput src, double beta = 0.5) - { - IHostEnvironment host = CatalogUtils.GetEnvironment(catalog); - - //check the root cause input - CheckRootCauseInput(host, src); - - //check beta - host.CheckUserArg(beta >= 0 && beta <= 1, nameof(beta), "Must be in [0,1]"); - - //find out the root cause - RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, beta); - RootCause dst = analyzer.Analyze(); - return dst; - } - - private static void CheckRootCauseInput(IHostEnvironment host, RootCauseLocalizationInput src) - { - host.CheckUserArg(src.Slices.Count >= 1, nameof(src.Slices), "Must has more than one item"); - - bool containsAnomalyTimestamp = false; - foreach (MetricSlice slice in src.Slices) - { - if (slice.TimeStamp.Equals(src.AnomalyTimestamp)) - { - containsAnomalyTimestamp = true; - } - } - host.CheckUserArg(containsAnomalyTimestamp, nameof(src.Slices), "Has no points in the given anomaly timestamp"); - } - /// /// Singular Spectrum Analysis (SSA) model for univariate time-series forecasting. /// For the details of the model, refer to http://arxiv.org/pdf/1206.6910.pdf. diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index b7112b1f8d..87a02b3cef 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -357,13 +357,11 @@ private void GetRootCauseDirectionAndScore(Dictionary d { dst.Items[i].Score = GetFinalScore(scoreList[i].Surprise, Math.Abs(scoreList[i].ExplanatoryScore), beta); } - } } } else if (dst.Items.Count == 1) { TimeSeriesPoint rootCausePoint = GetPointByDimension(dimPointMapping, dst.Items[0].Dimension, pointTree, aggType, aggSymbol); - Point rootCausePoint = GetPointByDimension(dimPointMapping, dst.Items[0].Dimension, pointTree, aggType, aggSymbol); if (anomalyPoint != null && rootCausePoint != null) { Tuple scores = GetSurpriseAndExplanatoryScore(rootCausePoint, anomalyPoint); @@ -416,7 +414,6 @@ private TimeSeriesPoint GetPointByDimension(Dictionary return p; } else - { { return null; } @@ -575,9 +572,6 @@ private double GetFinalScore(double surprise, double ep, double beta) b = (1 - Math.Pow(2, -ep)); } - return beta * a + (1 - beta) * b; - } - return beta * a + (1 - beta) * b; } @@ -740,7 +734,6 @@ public int CompareTo(object obj) return DimensionKey.CompareTo(other.DimensionKey); else throw new ArgumentException("Object is not a BestDimension"); - throw new ArgumentException("Object is not a BestDimension"); } } @@ -755,4 +748,4 @@ public RootCauseScore(double surprise, double explanatoryScore) ExplanatoryScore = explanatoryScore; } } -} +} \ No newline at end of file diff --git a/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs b/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs index 4bf1d4de22..b2c07ed855 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs @@ -10,46 +10,43 @@ namespace Microsoft.ML.TimeSeries { public sealed class RootCause { - /// /// A List for root cause item. Instance of the item should be - /// public List Items { get; set; } - public RootCause() - { - Items = new List(); - } - } + public RootCause() { Items = new List(); } + } public sealed class RootCauseLocalizationInput { - /// /// When the anomaly incident occurs - /// public DateTime AnomalyTimestamp { get; set; } - /// /// Point with the anomaly dimension must exist in the slice list at the anomaly timestamp, or the libary will not calculate the root cause /// public Dictionary AnomalyDimension { get; set; } - /// /// A list of points at different timestamp. If the slices don't contain point data corresponding to the anomaly timestamp, the root cause localization alogorithm will not calculate the root cause as no information at the anomaly timestamp is provided. - /// public List Slices { get; set; } - /// /// The aggregated type, the type should be /// public AggregateType AggregateType { get; set; } - + /// The string you defined as a aggregated symbol in the AnomalyDimension and point dimension. + public Object AggregateSymbol { get; set; } + /// + /// When the anomaly incident occurs + /// + /// + /// Point with the anomaly dimension must exist in the slice list at the anomaly timestamp, or the libary will not calculate the root cause + /// + /// + /// A list of points at different timestamp. If the slices don't contain point data corresponding to the anomaly timestamp, the root cause localization alogorithm will not calculate the root cause as no information at the anomaly timestamp is provided. + /// + /// + /// The aggregated type, the type should be + /// /// /// The string you defined as a aggregated symbol in the AnomalyDimension and point dimension. /// - public Object AggregateSymbol { get; set; } - //When the anomaly incident occurs - //Point with the anomaly dimension must exist in the slice list at the anomaly timestamp, or the libary will not calculate the root cause - //A list of points at different timestamp. If the slices don't contain point data corresponding to the anomaly timestamp, the root cause localization alogorithm will not calculate the root cause as no information at the anomaly timestamp is provided. - //The aggregated symbol in the AnomalyDimension and point dimension should be consistent public RootCauseLocalizationInput(DateTime anomalyTimestamp, Dictionary anomalyDimension, List slices, AggregateType aggregateType, Object aggregateSymbol) { diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index c5da84441d..8b30862d4f 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -8,8 +8,6 @@ using Microsoft.ML.Data; using Microsoft.ML.TestFramework; using Microsoft.ML.TimeSeries; -using Microsoft.ML.TestFrameworkCommon; -using Microsoft.ML.TimeSeries; using Microsoft.ML.Transforms.TimeSeries; using Xunit; using Xunit.Abstractions; @@ -530,10 +528,9 @@ public void AnomalyDetectionWithSrCnn(bool loadDataFromFile) { var ml = new MLContext(1); IDataView dataView; - if(loadDataFromFile) + if (loadDataFromFile) { var dataPath = GetDataPath(Path.Combine("Timeseries", "anomaly_detection.csv")); - // Load data from file into the dataView dataView = ml.Data.LoadFromTextFile(dataPath, new[] { new TextLoader.Column("Value", DataKind.Single, 0), @@ -580,44 +577,53 @@ public void AnomalyDetectionWithSrCnn(bool loadDataFromFile) [Theory, CombinatorialData] public void TestSrCnnBatchAnomalyDetector( - [CombinatorialValues(SrCnnDetectMode.AnomalyOnly, SrCnnDetectMode.AnomalyAndExpectedValue, SrCnnDetectMode.AnomalyAndMargin)]SrCnnDetectMode mode, - [CombinatorialValues(true, false)]bool loadDataFromFile, - [CombinatorialValues(-1, 24, 26, 512)]int batchSize) + [CombinatorialValues(SrCnnDetectMode.AnomalyOnly, SrCnnDetectMode.AnomalyAndExpectedValue, SrCnnDetectMode.AnomalyAndMargin)] SrCnnDetectMode mode, + [CombinatorialValues(true, false)] bool loadDataFromFile, + [CombinatorialValues(-1, 24, 26, 512)] int batchSize) + { var ml = new MLContext(1); IDataView dataView; if (loadDataFromFile) { var dataPath = GetDataPath("Timeseries", "anomaly_detection.csv"); + // Load data from file into the dataView dataView = ml.Data.LoadFromTextFile(dataPath, hasHeader: true); } else + { // Generate sample series data with an anomaly var data = new List(); for (int index = 0; index < 20; index++) { - data.Add(new TimeSeriesDataDouble { Value = 5 } ); + data.Add(new TimeSeriesDataDouble { Value = 5 }); } data.Add(new TimeSeriesDataDouble { Value = 10 }); for (int index = 0; index < 5; index++) { data.Add(new TimeSeriesDataDouble { Value = 5 }); } + // Convert data to IDataView. dataView = ml.Data.LoadFromEnumerable(data); } + // Setup the detection arguments string outputColumnName = nameof(SrCnnAnomalyDetection.Prediction); string inputColumnName = nameof(TimeSeriesDataDouble.Value); + // Do batch anomaly detection var outputDataView = ml.AnomalyDetection.DetectEntireAnomalyBySrCnn(dataView, outputColumnName, inputColumnName, threshold: 0.35, batchSize: batchSize, sensitivity: 90.0, mode); + // Getting the data of the newly created column as an IEnumerable of // SrCnnAnomalyDetection. var predictionColumn = ml.Data.CreateEnumerable( outputDataView, reuseRowObject: false); + int k = 0; foreach (var prediction in predictionColumn) + { switch (mode) { case SrCnnDetectMode.AnomalyOnly: @@ -654,108 +660,22 @@ public void TestSrCnnBatchAnomalyDetector( k += 1; } } + + [Fact] public void RootCauseLocalization() { // Create an root cause localizatiom input var rootCauseLocalizationInput = new RootCauseLocalizationInput(GetRootCauseTimestamp(), GetRootCauseAnomalyDimension(), new List() { new MetricSlice(GetRootCauseTimestamp(), GetRootCauseLocalizationPoints()) }, AggregateType.Sum, _rootCauseAggSymbol); + var ml = new MLContext(1); RootCause rootCause = ml.AnomalyDetection.LocalizeRootCause(rootCauseLocalizationInput); + Assert.NotNull(rootCause); Assert.Equal(1, (int)rootCause.Items.Count); Assert.Equal(3, (int)rootCause.Items[0].Dimension.Count); Assert.Equal(AnomalyDirection.Up, rootCause.Items[0].Direction); Assert.Equal(1, (int)rootCause.Items[0].Path.Count); Assert.Equal("DataCenter", rootCause.Items[0].Path[0]); - Dictionary expectedDim = new Dictionary(); - expectedDim.Add("Country", "UK"); - expectedDim.Add("DeviceType", _rootCauseAggSymbol); - expectedDim.Add("DataCenter", "DC1"); - foreach (KeyValuePair pair in rootCause.Items[0].Dimension) - { - Assert.Equal(expectedDim[pair.Key], pair.Value); - } - var newRootCauseInput = new RootCauseLocalizationData(timeStamp, GetAnomalyDimension(), new List() { new MetricSlice(timeStamp, GetRootCauseLocalizationPoints()) }, AggregateType.Sum, _aggSymbol); - Dictionary expectedDim = new Dictionary(); - expectedDim.Add("Country","UK"); - expectedDim.Add("DeviceType",_aggSymbol); - expectedDim.Add("DataCenter","DC1"); - - foreach (KeyValuePair pair in transformedRootCause.RootCause.Items[0].Dimension) { - Assert.Equal(expectedDim[pair.Key], pair.Value); - } - } - private static List GetRootCauseLocalizationPoints() - { - List points = new List(); - Dictionary dic1 = new Dictionary(); - dic1.Add("Country", "UK"); - dic1.Add("DeviceType", "Laptop"); - dic1.Add("DataCenter", "DC1"); - points.Add(new TimeSeriesPoint(200, 100, true, dic1)); - Dictionary dic2 = new Dictionary(); - dic2.Add("Country", "UK"); - dic2.Add("DeviceType", "Mobile"); - dic2.Add("DataCenter", "DC1"); - points.Add(new TimeSeriesPoint(1000, 100, true, dic2)); - Dictionary dic3 = new Dictionary(); - dic3.Add("Country", "UK"); - dic3.Add("DeviceType", _rootCauseAggSymbol); - dic3.Add("DataCenter", "DC1"); - points.Add(new TimeSeriesPoint(1200, 200, true, dic3)); - Dictionary dic4 = new Dictionary(); - dic4.Add("Country", "UK"); - dic4.Add("DeviceType", "Laptop"); - dic4.Add("DataCenter", "DC2"); - points.Add(new TimeSeriesPoint(100, 100, false, dic4)); - Dictionary dic5 = new Dictionary(); - dic5.Add("Country", "UK"); - dic5.Add("DeviceType", "Mobile"); - dic5.Add("DataCenter", "DC2"); - points.Add(new TimeSeriesPoint(200, 200, false, dic5)); - Dictionary dic6 = new Dictionary(); - dic6.Add("Country", "UK"); - dic6.Add("DeviceType", _rootCauseAggSymbol); - dic6.Add("DataCenter", "DC2"); - points.Add(new TimeSeriesPoint(300, 300, false, dic6)); - Dictionary dic7 = new Dictionary(); - dic7.Add("Country", "UK"); - dic7.Add("DeviceType", _rootCauseAggSymbol); - dic7.Add("DataCenter", _rootCauseAggSymbol); - points.Add(new TimeSeriesPoint(1500, 500, true, dic7)); - Dictionary dic8 = new Dictionary(); - dic8.Add("Country", "UK"); - dic8.Add("DeviceType", "Laptop"); - dic8.Add("DataCenter", _rootCauseAggSymbol); - points.Add(new TimeSeriesPoint(300, 200, true, dic8)); - Dictionary dic9 = new Dictionary(); - dic9.Add("Country", "UK"); - dic9.Add("DeviceType", "Mobile"); - dic9.Add("DataCenter", _rootCauseAggSymbol); - points.Add(new TimeSeriesPoint(1200, 300, true, dic9)); - return points; - private static Dictionary GetRootCauseAnomalyDimension() - Dictionary dim = new Dictionary(); - dim.Add("Country", "UK"); - dim.Add("DeviceType", _rootCauseAggSymbol); - dim.Add("DataCenter", _rootCauseAggSymbol); - return dim; - } - private static DateTime GetRootCauseTimestamp() - { - return new DateTime(2020, 3, 23, 0, 0, 0); - return points; - } - private static Dictionary GetAnomalyDimension() - { - Dictionary dim = new Dictionary(); - dim.Add("Country", "UK"); - dim.Add("DeviceType", _aggSymbol); - dim.Add("DataCenter", _aggSymbol); - return dim; - } - private static DateTime GetCurrentTimestamp() - { - return new DateTime(); Dictionary expectedDim = new Dictionary(); expectedDim.Add("Country", "UK"); @@ -767,136 +687,64 @@ private static DateTime GetCurrentTimestamp() Assert.Equal(expectedDim[pair.Key], pair.Value); } } - private static List GetRootCauseLocalizationPoints() - { - List points = new List(); - Dictionary dic1 = new Dictionary(); - dic1.Add("Country", "UK"); - dic1.Add("DeviceType", "Laptop"); - dic1.Add("DataCenter", "DC1"); - points.Add(new Point(200, 100, true, dic1)); - Dictionary dic2 = new Dictionary(); - dic2.Add("Country", "UK"); - dic2.Add("DeviceType", "Mobile"); - dic2.Add("DataCenter", "DC1"); - points.Add(new Point(1000, 100, true, dic2)); - Dictionary dic3 = new Dictionary(); - dic3.Add("Country", "UK"); - dic3.Add("DeviceType", _aggSymbol); - dic3.Add("DataCenter", "DC1"); - points.Add(new Point(1200, 200, true, dic3)); - Dictionary dic4 = new Dictionary(); - dic4.Add("Country", "UK"); - dic4.Add("DeviceType", "Laptop"); - dic4.Add("DataCenter", "DC2"); - points.Add(new Point(100, 100, false, dic4)); - - Dictionary dic5 = new Dictionary(); - dic5.Add("Country", "UK"); - dic5.Add("DeviceType", "Mobile"); - dic5.Add("DataCenter", "DC2"); - points.Add(new Point(200, 200, false, dic5)); - - Dictionary dic6 = new Dictionary(); - dic6.Add("Country", "UK"); - dic6.Add("DeviceType", _aggSymbol); - dic6.Add("DataCenter", "DC2"); - points.Add(new Point(300, 300, false, dic6)); - - Dictionary dic7 = new Dictionary(); - dic7.Add("Country", "UK"); - dic7.Add("DeviceType", _aggSymbol); - dic7.Add("DataCenter", _aggSymbol); - points.Add(new Point(1500, 500, true, dic7)); - - Dictionary dic8 = new Dictionary(); - dic8.Add("Country", "UK"); - dic8.Add("DeviceType", "Laptop"); - dic8.Add("DataCenter", _aggSymbol); - points.Add(new Point(300, 200, true, dic8)); - - Dictionary dic9 = new Dictionary(); - dic9.Add("Country", "UK"); - dic9.Add("DeviceType", "Mobile"); - dic9.Add("DataCenter", _aggSymbol); - points.Add(new Point(1200, 300, true, dic9)); - - return points; - } - - private static Dictionary GetAnomalyDimension() - { - Dictionary dim = new Dictionary(); - dim.Add("Country", "UK"); - dim.Add("DeviceType", _aggSymbol); - dim.Add("DataCenter", _aggSymbol); - - return dim; - } - - private static DateTime GetCurrentTimestamp() - { - return new DateTime(2020, 3, 23, 0, 0, 0); - } - - private static List GetRootCauseLocalizationPoints() + private static List GetRootCauseLocalizationPoints() { - List points = new List(); + List points = new List(); Dictionary dic1 = new Dictionary(); dic1.Add("Country", "UK"); dic1.Add("DeviceType", "Laptop"); dic1.Add("DataCenter", "DC1"); - points.Add(new Point(200, 100, true, dic1)); + points.Add(new TimeSeriesPoint(200, 100, true, dic1)); Dictionary dic2 = new Dictionary(); dic2.Add("Country", "UK"); dic2.Add("DeviceType", "Mobile"); dic2.Add("DataCenter", "DC1"); - points.Add(new Point(1000, 100, true, dic2)); + points.Add(new TimeSeriesPoint(1000, 100, true, dic2)); Dictionary dic3 = new Dictionary(); dic3.Add("Country", "UK"); dic3.Add("DeviceType", _rootCauseAggSymbol); dic3.Add("DataCenter", "DC1"); - points.Add(new Point(1200, 200, true, dic3)); + points.Add(new TimeSeriesPoint(1200, 200, true, dic3)); Dictionary dic4 = new Dictionary(); dic4.Add("Country", "UK"); dic4.Add("DeviceType", "Laptop"); dic4.Add("DataCenter", "DC2"); - points.Add(new Point(100, 100, false, dic4)); + points.Add(new TimeSeriesPoint(100, 100, false, dic4)); Dictionary dic5 = new Dictionary(); dic5.Add("Country", "UK"); dic5.Add("DeviceType", "Mobile"); dic5.Add("DataCenter", "DC2"); - points.Add(new Point(200, 200, false, dic5)); + points.Add(new TimeSeriesPoint(200, 200, false, dic5)); Dictionary dic6 = new Dictionary(); dic6.Add("Country", "UK"); dic6.Add("DeviceType", _rootCauseAggSymbol); dic6.Add("DataCenter", "DC2"); - points.Add(new Point(300, 300, false, dic6)); + points.Add(new TimeSeriesPoint(300, 300, false, dic6)); Dictionary dic7 = new Dictionary(); dic7.Add("Country", "UK"); dic7.Add("DeviceType", _rootCauseAggSymbol); dic7.Add("DataCenter", _rootCauseAggSymbol); - points.Add(new Point(1500, 500, true, dic7)); + points.Add(new TimeSeriesPoint(1500, 500, true, dic7)); Dictionary dic8 = new Dictionary(); dic8.Add("Country", "UK"); dic8.Add("DeviceType", "Laptop"); dic8.Add("DataCenter", _rootCauseAggSymbol); - points.Add(new Point(300, 200, true, dic8)); + points.Add(new TimeSeriesPoint(300, 200, true, dic8)); Dictionary dic9 = new Dictionary(); dic9.Add("Country", "UK"); dic9.Add("DeviceType", "Mobile"); dic9.Add("DataCenter", _rootCauseAggSymbol); - points.Add(new Point(1200, 300, true, dic9)); + points.Add(new TimeSeriesPoint(1200, 300, true, dic9)); return points; } From a3eee1a8bc2010cb5961d86e52e396b735752551 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Wed, 3 Jun 2020 17:52:21 +0800 Subject: [PATCH 28/36] update --- .../time-series-root-cause-localization.md | 6 ++-- .../ExtensionsCatalog.cs | 2 -- .../RootCauseLocalizationType.cs | 30 +++++++++---------- .../TimeSeriesDirectApi.cs | 8 ++--- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/docs/api-reference/time-series-root-cause-localization.md b/docs/api-reference/time-series-root-cause-localization.md index 3eecc838e2..47a4132a18 100644 --- a/docs/api-reference/time-series-root-cause-localization.md +++ b/docs/api-reference/time-series-root-cause-localization.md @@ -1,4 +1,4 @@ -At Mircosoft, we develop a decision tree based root cause localization method which helps to find out the root causes for an anomaly incident at a specific timestamp incrementally. +At Microsoft, we have developed a decision tree based root cause localization method which helps to find out the root causes for an anomaly incident at a specific timestamp incrementally. ## Multi-Dimensional Root Cause Localization It's a common case that one measure is collected with many dimensions (*e.g.*, Province, ISP) whose values are categorical(*e.g.*, Beijing or Shanghai for dimension Province). When a measure's value deviates from its expected value, this measure encounters anomalies. In such case, operators would like to localize the root cause dimension combinations rapidly and accurately. Multi-dimensional root cause localization is critical to troubleshoot and mitigate such case. @@ -13,7 +13,7 @@ The decision tree based root cause localization method is unsupervised, which me ### Decision Tree -[Decision tree](https://en.wikipedia.org/wiki/Decision_tree) algorithm chooses the highest information gain to split or construct a decision tree.  We use it to choose the dimension which contributes the most to the anomaly. Following are some concepts used in decision tree. +The [Decision tree](https://en.wikipedia.org/wiki/Decision_tree) algorithm chooses the highest information gain to split or construct a decision tree.  We use it to choose the dimension which contributes the most to the anomaly. Below are some concepts used in decision trees. #### Information Entropy @@ -30,7 +30,7 @@ $$Gain(D, a) = Ent(D) - \sum_{v=1}^{|V|} \frac{|D^V|}{|D |} Ent(D^v) $$ Where $Ent(D^v)$ is the entropy of set points in D for which dimension $a$ is equal to $v$, $|D|$ is the total number of points in dataset $D$. $|D^V|$ is the total number of points in dataset $D$ for which dimension $a$ is equal to $v$. -For all aggregated dimensions, we calculate the information for each dimension. The greater the reduction in this uncertainty, the more information is gained about D from dimension $a$. +For all aggregated dimensions, we calculate the information for each dimension. The greater the reduction in this uncertainty, the more information is gained about $D$ from dimension $a$. #### Entropy Gain Ratio diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 7cba0ee711..161f8bc27b 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -2,8 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; -using System.Reflection; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TimeSeries; diff --git a/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs b/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs index b2c07ed855..3a823f5e73 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs @@ -10,43 +10,41 @@ namespace Microsoft.ML.TimeSeries { public sealed class RootCause { + /// /// A List for root cause item. Instance of the item should be + /// public List Items { get; set; } - public RootCause() { Items = new List(); } - + public RootCause() + { + Items = new List(); + } } public sealed class RootCauseLocalizationInput { + /// /// When the anomaly incident occurs + /// public DateTime AnomalyTimestamp { get; set; } + /// /// Point with the anomaly dimension must exist in the slice list at the anomaly timestamp, or the libary will not calculate the root cause /// public Dictionary AnomalyDimension { get; set; } + /// /// A list of points at different timestamp. If the slices don't contain point data corresponding to the anomaly timestamp, the root cause localization alogorithm will not calculate the root cause as no information at the anomaly timestamp is provided. + /// public List Slices { get; set; } + /// /// The aggregated type, the type should be /// public AggregateType AggregateType { get; set; } - /// The string you defined as a aggregated symbol in the AnomalyDimension and point dimension. - public Object AggregateSymbol { get; set; } - /// - /// When the anomaly incident occurs - /// - /// - /// Point with the anomaly dimension must exist in the slice list at the anomaly timestamp, or the libary will not calculate the root cause - /// - /// - /// A list of points at different timestamp. If the slices don't contain point data corresponding to the anomaly timestamp, the root cause localization alogorithm will not calculate the root cause as no information at the anomaly timestamp is provided. - /// - /// - /// The aggregated type, the type should be - /// + /// /// The string you defined as a aggregated symbol in the AnomalyDimension and point dimension. /// + public Object AggregateSymbol { get; set; } public RootCauseLocalizationInput(DateTime anomalyTimestamp, Dictionary anomalyDimension, List slices, AggregateType aggregateType, Object aggregateSymbol) { diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index 8b30862d4f..a6d344f910 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -99,8 +99,6 @@ private sealed class SrCnnAnomalyDetection private static Object _rootCauseAggSymbol = "##SUM##"; - - [Fact] public void ChangeDetection() { @@ -577,9 +575,9 @@ public void AnomalyDetectionWithSrCnn(bool loadDataFromFile) [Theory, CombinatorialData] public void TestSrCnnBatchAnomalyDetector( - [CombinatorialValues(SrCnnDetectMode.AnomalyOnly, SrCnnDetectMode.AnomalyAndExpectedValue, SrCnnDetectMode.AnomalyAndMargin)] SrCnnDetectMode mode, - [CombinatorialValues(true, false)] bool loadDataFromFile, - [CombinatorialValues(-1, 24, 26, 512)] int batchSize) + [CombinatorialValues(SrCnnDetectMode.AnomalyOnly, SrCnnDetectMode.AnomalyAndExpectedValue, SrCnnDetectMode.AnomalyAndMargin)] SrCnnDetectMode mode, + [CombinatorialValues(true, false)] bool loadDataFromFile, + [CombinatorialValues(-1, 24, 26, 512)] int batchSize) { var ml = new MLContext(1); IDataView dataView; From 66261e68f91ee0fb21347cac3d634796316f54e6 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Tue, 9 Jun 2020 17:55:25 +0800 Subject: [PATCH 29/36] some update --- .../time-series-root-cause-localization.md | 2 +- .../ExtensionsCatalog.cs | 5 +- .../RootCauseAnalyzer.cs | 59 +++++++++++++++---- .../RootCauseLocalizationType.cs | 9 ++- .../TimeSeriesDirectApi.cs | 7 ++- 5 files changed, 63 insertions(+), 19 deletions(-) diff --git a/docs/api-reference/time-series-root-cause-localization.md b/docs/api-reference/time-series-root-cause-localization.md index 47a4132a18..d3336b378a 100644 --- a/docs/api-reference/time-series-root-cause-localization.md +++ b/docs/api-reference/time-series-root-cause-localization.md @@ -1,7 +1,7 @@ At Microsoft, we have developed a decision tree based root cause localization method which helps to find out the root causes for an anomaly incident at a specific timestamp incrementally. ## Multi-Dimensional Root Cause Localization -It's a common case that one measure is collected with many dimensions (*e.g.*, Province, ISP) whose values are categorical(*e.g.*, Beijing or Shanghai for dimension Province). When a measure's value deviates from its expected value, this measure encounters anomalies. In such case, operators would like to localize the root cause dimension combinations rapidly and accurately. Multi-dimensional root cause localization is critical to troubleshoot and mitigate such case. +It's a common case that one measure is collected with many dimensions (*e.g.*, Province, ISP) whose values are categorical(*e.g.*, Beijing or Shanghai for dimension Province). When a measure's value deviates from its expected value, this measure encounters anomalies. In such case, users would like to localize the root cause dimension combinations rapidly and accurately. Multi-dimensional root cause localization is critical to troubleshoot and mitigate such case. ## Algorithm diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 161f8bc27b..cc72cca4fc 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -187,6 +187,7 @@ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog /// It is used when score is calculated for each root cause item. The range of beta should be in [0,1]. /// For a larger beta, root cause items which have a large difference between value and expected value will get a high score. /// For a small beta, root cause items which have a high relative change will get a low score. + /// A threshold to determine whether the point should be root cause. If the point's delta is equal to or larger than anomalyDeltaThreshold multiplies anomaly dimension point's delta, this point is treated as a root cause. Different threshold will turn out different result. Users can choose the delta according to their data and requirment. /// /// /// /// /// - public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, RootCauseLocalizationInput src, double beta = 0.5) + public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, RootCauseLocalizationInput src, double beta = 0.5, double rootCauseThreshold = 0.95) { IHostEnvironment host = CatalogUtils.GetEnvironment(catalog); @@ -205,7 +206,7 @@ public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, host.CheckUserArg(beta >= 0 && beta <= 1, nameof(beta), "Must be in [0,1]"); //find out the root cause - RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, beta); + RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, beta, rootCauseThreshold); RootCause dst = analyzer.Analyze(); return dst; } diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index 87a02b3cef..22cb6fa916 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -13,16 +13,17 @@ namespace Microsoft.ML.TimeSeries public class RootCauseAnalyzer { private static double _anomalyRatioThreshold = 0.5; - private static double _anomalyDeltaThreshold = 0.95; private static double _anomalyPreDeltaThreshold = 2; private RootCauseLocalizationInput _src; private double _beta; + private double _rootCauseThreshold; - public RootCauseAnalyzer(RootCauseLocalizationInput src, double beta) + public RootCauseAnalyzer(RootCauseLocalizationInput src, double beta, double rootCauseThreshold) { _src = src; _beta = beta; + _rootCauseThreshold = rootCauseThreshold; } public RootCause Analyze() @@ -421,7 +422,7 @@ private TimeSeriesPoint GetPointByDimension(Dictionary private static string GetDicCode(Dictionary dic) { - return string.Join(";", dic.Select(x => x.Key + "=" + (string)x.Value).ToArray()); + return string.Join(";", dic.Select(x => x.Key + "=" + Convert.ToString(x.Value)).ToArray()); } private void BuildTree(PointTree tree, List aggDims, TimeSeriesPoint point, Object aggSymbol) @@ -476,25 +477,39 @@ private BestDimension FindBestDimension(SortedDictionary bool isRatioNan = Double.IsNaN(valueRatioMap[best]); if (dimension.Key.AnomalyDis.Count > 1) { - if (!isRatioNan && (best.AnomalyDis.Count != 1 && (isLeavesLevel ? valueRatioMap[best].CompareTo(dimension.Value) <= 0 : valueRatioMap[best].CompareTo(dimension.Value) >= 0))) + if (best.AnomalyDis.Count != 1 && !isRatioNan && (isLeavesLevel ? valueRatioMap[best].CompareTo(dimension.Value) <= 0 : valueRatioMap[best].CompareTo(dimension.Value) >= 0)) { - best = dimension.Key; + best = GetBestDimension(best, dimension, valueRatioMap); } } - else + else if (dimension.Key.AnomalyDis.Count == 1) { + if (best.AnomalyDis.Count > 1) { best = dimension.Key; } - else + else if (best.AnomalyDis.Count == 1) { if (!isRatioNan && (isLeavesLevel ? valueRatioMap[best].CompareTo(dimension.Value) <= 0 : valueRatioMap[best].CompareTo(dimension.Value) >= 0)) { - best = dimension.Key; + best = GetBestDimension(best, dimension, valueRatioMap); } } } + //{ + // if (best.AnomalyDis.Count > 1) + // { + // best = dimension.Key; + // } + // else + // { + // if (!isRatioNan && (isLeavesLevel ? valueRatioMap[best].CompareTo(dimension.Value) <= 0 : valueRatioMap[best].CompareTo(dimension.Value) >= 0)) + // { + // best = dimension.Key; + // } + // } + //} } } } @@ -502,6 +517,22 @@ private BestDimension FindBestDimension(SortedDictionary return best; } + private BestDimension GetBestDimension(BestDimension best, KeyValuePair dimension, Dictionary valueRatioMap) + { + if (valueRatioMap[best].CompareTo(dimension.Value) == 0) + { + if (dimension.Key.AnomalyDis.Count != dimension.Key.PointDis.Count) + { + best = dimension.Key; + } + } + else + { + best = dimension.Key; + } + return best; + } + /// /// Calculate the surprise score according to root cause point and anomaly point /// @@ -569,6 +600,10 @@ private double GetFinalScore(double surprise, double ep, double beta) else { a = (1 - Math.Pow(2, -surprise)); + if (Double.IsNaN(a)) + { + a = 1; + } b = (1 - Math.Pow(2, -ep)); } @@ -593,7 +628,7 @@ private static Dictionary UpdateDimensionValue(Dictionary distribution, List bigDictionary, Dictio private bool IsAggregationDimension(Object val, Object aggSymbol) { - return val.Equals(aggSymbol); + return Convert.ToString(val).Equals(aggSymbol); } } diff --git a/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs b/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs index 3a823f5e73..3b2b311d2a 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs @@ -19,6 +19,7 @@ public RootCause() Items = new List(); } } + public sealed class RootCauseLocalizationInput { /// @@ -55,7 +56,7 @@ public RootCauseLocalizationInput(DateTime anomalyTimestamp, Dictionary anomalyDimension, List slices, string aggregateSymbol) + public RootCauseLocalizationInput(DateTime anomalyTimestamp, Dictionary anomalyDimension, List slices, Object aggregateSymbol) { AnomalyTimestamp = anomalyTimestamp; AnomalyDimension = anomalyDimension; @@ -63,6 +64,8 @@ public RootCauseLocalizationInput(DateTime anomalyTimestamp, Dictionary points) TimeStamp = timeStamp; Points = points; } + + public MetricSlice() { } } public sealed class TimeSeriesPoint : IEquatable @@ -197,6 +202,8 @@ public TimeSeriesPoint(Dictionary dimension) { Dimension = dimension; } + public TimeSeriesPoint() { } + public TimeSeriesPoint(double value, double expectedValue, bool isAnomaly, Dictionary dimension) { Value = value; diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index a6d344f910..566305d670 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -529,6 +529,7 @@ public void AnomalyDetectionWithSrCnn(bool loadDataFromFile) if (loadDataFromFile) { var dataPath = GetDataPath(Path.Combine("Timeseries", "anomaly_detection.csv")); + // Load data from file into the dataView dataView = ml.Data.LoadFromTextFile(dataPath, new[] { new TextLoader.Column("Value", DataKind.Single, 0), @@ -575,9 +576,9 @@ public void AnomalyDetectionWithSrCnn(bool loadDataFromFile) [Theory, CombinatorialData] public void TestSrCnnBatchAnomalyDetector( - [CombinatorialValues(SrCnnDetectMode.AnomalyOnly, SrCnnDetectMode.AnomalyAndExpectedValue, SrCnnDetectMode.AnomalyAndMargin)] SrCnnDetectMode mode, - [CombinatorialValues(true, false)] bool loadDataFromFile, - [CombinatorialValues(-1, 24, 26, 512)] int batchSize) + [CombinatorialValues(SrCnnDetectMode.AnomalyOnly, SrCnnDetectMode.AnomalyAndExpectedValue, SrCnnDetectMode.AnomalyAndMargin)] SrCnnDetectMode mode, + [CombinatorialValues(true, false)] bool loadDataFromFile, + [CombinatorialValues(-1, 24, 26, 512)] int batchSize) { var ml = new MLContext(1); IDataView dataView; From 887e99271653da2e74c43726cfc43b5a852dfe78 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Tue, 9 Jun 2020 18:14:41 +0800 Subject: [PATCH 30/36] update --- src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs | 13 ------------- .../TimeSeriesDirectApi.cs | 1 + 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index 22cb6fa916..fd79f49bc4 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -497,19 +497,6 @@ private BestDimension FindBestDimension(SortedDictionary } } } - //{ - // if (best.AnomalyDis.Count > 1) - // { - // best = dimension.Key; - // } - // else - // { - // if (!isRatioNan && (isLeavesLevel ? valueRatioMap[best].CompareTo(dimension.Value) <= 0 : valueRatioMap[best].CompareTo(dimension.Value) >= 0)) - // { - // best = dimension.Key; - // } - // } - //} } } } diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index 566305d670..406d0c879f 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -99,6 +99,7 @@ private sealed class SrCnnAnomalyDetection private static Object _rootCauseAggSymbol = "##SUM##"; + [Fact] public void ChangeDetection() { From 421157d81392fff1838e802ca52a772ed29e209e Mon Sep 17 00:00:00 2001 From: Shakira Sun <61308292+suxi-ms@users.noreply.github.com> Date: Thu, 11 Jun 2020 09:19:36 +0800 Subject: [PATCH 31/36] Update src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs Co-authored-by: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> --- src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index cc72cca4fc..981125801c 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -187,7 +187,7 @@ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog /// It is used when score is calculated for each root cause item. The range of beta should be in [0,1]. /// For a larger beta, root cause items which have a large difference between value and expected value will get a high score. /// For a small beta, root cause items which have a high relative change will get a low score. - /// A threshold to determine whether the point should be root cause. If the point's delta is equal to or larger than anomalyDeltaThreshold multiplies anomaly dimension point's delta, this point is treated as a root cause. Different threshold will turn out different result. Users can choose the delta according to their data and requirment. + /// A threshold to determine whether the point should be root cause. If the point's delta is equal to or larger than anomalyDeltaThreshold multiplied by anomaly dimension point's delta, this point is treated as a root cause. Different threshold will turn out different results. Users can choose the delta according to their data and requirements. /// /// /// Date: Thu, 11 Jun 2020 09:21:22 +0800 Subject: [PATCH 32/36] fix typo in extension catalgo comments --- src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index cc72cca4fc..ab080cff55 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -187,7 +187,7 @@ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog /// It is used when score is calculated for each root cause item. The range of beta should be in [0,1]. /// For a larger beta, root cause items which have a large difference between value and expected value will get a high score. /// For a small beta, root cause items which have a high relative change will get a low score. - /// A threshold to determine whether the point should be root cause. If the point's delta is equal to or larger than anomalyDeltaThreshold multiplies anomaly dimension point's delta, this point is treated as a root cause. Different threshold will turn out different result. Users can choose the delta according to their data and requirment. + /// A threshold to determine whether the point should be root cause. If the point's delta is equal to or larger than rootCauseThreshold multiplied by anomaly dimension point's delta, this point is treated as a root cause. Different threshold will turn out different results. Users can choose the delta according to their data and requirments. /// /// /// Date: Fri, 12 Jun 2020 17:19:42 +0800 Subject: [PATCH 33/36] update libmf --- src/Native/MatrixFactorizationNative/libmf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Native/MatrixFactorizationNative/libmf b/src/Native/MatrixFactorizationNative/libmf index 298715a4e4..403153ca20 160000 --- a/src/Native/MatrixFactorizationNative/libmf +++ b/src/Native/MatrixFactorizationNative/libmf @@ -1 +1 @@ -Subproject commit 298715a4e458bc09c6a27c8643a58095afbdadf1 +Subproject commit 403153ca204817e2901b2872d977088316360641 From cd1988927c09764d635aec87c333420a679b4bf7 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Fri, 12 Jun 2020 22:21:40 +0800 Subject: [PATCH 34/36] update point map --- .../RootCauseAnalyzer.cs | 71 ++++++++++++++----- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index fd79f49bc4..bc18c0b248 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -40,10 +40,10 @@ private RootCause AnalyzeOneLayer(RootCauseLocalizationInput src) dst.Items = new List(); DimensionInfo dimensionInfo = SeparateDimension(src.AnomalyDimension, src.AggregateSymbol); - Tuple> pointInfo = GetPointsInfo(src, dimensionInfo); + Tuple, TimeSeriesPoint>> pointInfo = GetPointsInfo(src, dimensionInfo); PointTree pointTree = pointInfo.Item1; PointTree anomalyTree = pointInfo.Item2; - Dictionary dimPointMapping = pointInfo.Item3; + Dictionary, TimeSeriesPoint> dimPointMapping = pointInfo.Item3; //which means there is no anomaly point with the anomaly dimension or no point under anomaly dimension if (anomalyTree.ParentNode == null || dimPointMapping.Count == 0) @@ -82,11 +82,12 @@ private DimensionInfo SeparateDimension(Dictionary dimensions, O return info; } - private Tuple> GetPointsInfo(RootCauseLocalizationInput src, DimensionInfo dimensionInfo) + private Tuple, TimeSeriesPoint>> GetPointsInfo(RootCauseLocalizationInput src, DimensionInfo dimensionInfo) { PointTree pointTree = new PointTree(); PointTree anomalyTree = new PointTree(); - Dictionary dimPointMapping = new Dictionary(); + DimensionComparer dc = new DimensionComparer(); + Dictionary, TimeSeriesPoint> dimPointMapping = new Dictionary, TimeSeriesPoint>(dc); List totalPoints = GetTotalPointsForAnomalyTimestamp(src); Dictionary subDim = GetSubDim(src.AnomalyDimension, dimensionInfo.DetailDims); @@ -95,9 +96,9 @@ private Tuple> GetPoin { if (ContainsAll(point.Dimension, subDim)) { - if (!dimPointMapping.ContainsKey(GetDicCode(point.Dimension))) + if (!dimPointMapping.ContainsKey(point.Dimension)) { - dimPointMapping.Add(GetDicCode(point.Dimension), point); + dimPointMapping.Add(point.Dimension, point); bool isValidPoint = point.IsAnomaly == true; if (ContainsAll(point.Dimension, subDim)) { @@ -112,7 +113,7 @@ private Tuple> GetPoin } } - return new Tuple>(pointTree, anomalyTree, dimPointMapping); + return new Tuple, TimeSeriesPoint>>(pointTree, anomalyTree, dimPointMapping); } protected Dictionary GetSubDim(Dictionary dimension, List keyList) @@ -328,7 +329,7 @@ private AnomalyDirection GetRootCauseDirection(TimeSeriesPoint rootCausePoint) } } - private void GetRootCauseDirectionAndScore(Dictionary dimPointMapping, Dictionary anomalyRoot, RootCause dst, double beta, PointTree pointTree, AggregateType aggType, Object aggSymbol) + private void GetRootCauseDirectionAndScore(Dictionary, TimeSeriesPoint> dimPointMapping, Dictionary anomalyRoot, RootCause dst, double beta, PointTree pointTree, AggregateType aggType, Object aggSymbol) { TimeSeriesPoint anomalyPoint = GetPointByDimension(dimPointMapping, anomalyRoot, pointTree, aggType, aggSymbol); if (dst.Items.Count > 1) @@ -379,11 +380,11 @@ private void GetRootCauseDirectionAndScore(Dictionary d } } - private TimeSeriesPoint GetPointByDimension(Dictionary dimPointMapping, Dictionary dimension, PointTree pointTree, AggregateType aggType, Object aggSymbol) + private TimeSeriesPoint GetPointByDimension(Dictionary, TimeSeriesPoint> dimPointMapping, Dictionary dimension, PointTree pointTree, AggregateType aggType, Object aggSymbol) { - if (dimPointMapping.ContainsKey(GetDicCode(dimension))) + if (dimPointMapping.ContainsKey(dimension)) { - return dimPointMapping[GetDicCode(dimension)]; + return dimPointMapping[dimension]; } int count = 0; @@ -420,11 +421,6 @@ private TimeSeriesPoint GetPointByDimension(Dictionary } } - private static string GetDicCode(Dictionary dic) - { - return string.Join(";", dic.Select(x => x.Key + "=" + Convert.ToString(x.Value)).ToArray()); - } - private void BuildTree(PointTree tree, List aggDims, TimeSeriesPoint point, Object aggSymbol) { int aggNum = 0; @@ -770,4 +766,47 @@ public RootCauseScore(double surprise, double explanatoryScore) ExplanatoryScore = explanatoryScore; } } + + internal class DimensionComparer : EqualityComparer> + { + public override bool Equals(Dictionary x, Dictionary y) + { + if (x == null && y == null) + { + return true; + } + if ((x == null && y != null) || (x != null && y == null)) + { + return false; + } + if (x.Count != y.Count) + { + return false; + } + if (x.Keys.Except(y.Keys).Any()) + { + return false; + } + if (y.Keys.Except(x.Keys).Any()) + { + return false; + } + foreach (var pair in x) + { + if (!pair.Value.Equals(y[pair.Key])) + { + return false; + } + } + return true; + } + + public override int GetHashCode(Dictionary obj) + { + int code = 0; + foreach (KeyValuePair pair in obj) + code = code ^ pair.GetHashCode(); + return code; + } + } } \ No newline at end of file From 28bf0a531096a56786d16fccc73ab5e9d792b55d Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Thu, 18 Jun 2020 20:32:14 +0800 Subject: [PATCH 35/36] add root cause options --- .../ExtensionsCatalog.cs | 16 +++++------- .../RootCauseAnalyzer.cs | 26 ++++++++++++++++--- .../RootCauseLocalizationType.cs | 22 ++++++++-------- 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index ab080cff55..672ede75ba 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -183,11 +183,7 @@ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog /// /// The anomaly detection catalog. /// Root cause's input. The data is an instance of . - /// Beta is a weight parameter for user to choose. - /// It is used when score is calculated for each root cause item. The range of beta should be in [0,1]. - /// For a larger beta, root cause items which have a large difference between value and expected value will get a high score. - /// For a small beta, root cause items which have a high relative change will get a low score. - /// A threshold to determine whether the point should be root cause. If the point's delta is equal to or larger than rootCauseThreshold multiplied by anomaly dimension point's delta, this point is treated as a root cause. Different threshold will turn out different results. Users can choose the delta according to their data and requirments. + /// An instance of of to indicate root cause parameters. /// /// /// /// /// - public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, RootCauseLocalizationInput src, double beta = 0.5, double rootCauseThreshold = 0.95) + public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, RootCauseLocalizationInput src, RootCauseOptions options = null) { + options = options ?? new RootCauseOptions(); IHostEnvironment host = CatalogUtils.GetEnvironment(catalog); //check the root cause input CheckRootCauseInput(host, src); - //check beta - host.CheckUserArg(beta >= 0 && beta <= 1, nameof(beta), "Must be in [0,1]"); + //check parameters + host.CheckUserArg(options.Beta >= 0 && options.Beta <= 1, nameof(options.Beta), "Must be in [0,1]"); + host.CheckUserArg(options.RootCauseThreshold >= 0 && options.RootCauseThreshold <= 1, nameof(options.Beta), "Must be in [0,1]"); //find out the root cause - RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, beta, rootCauseThreshold); + RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, options); RootCause dst = analyzer.Analyze(); return dst; } diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index bc18c0b248..f2c70225a6 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -19,11 +19,11 @@ public class RootCauseAnalyzer private double _beta; private double _rootCauseThreshold; - public RootCauseAnalyzer(RootCauseLocalizationInput src, double beta, double rootCauseThreshold) + public RootCauseAnalyzer(RootCauseLocalizationInput src, RootCauseOptions options) { _src = src; - _beta = beta; - _rootCauseThreshold = rootCauseThreshold; + _beta = options.Beta; + _rootCauseThreshold = options.RootCauseThreshold; } public RootCause Analyze() @@ -755,6 +755,26 @@ public int CompareTo(object obj) } } + public class RootCauseOptions + { + internal double Beta; + /// + /// A threshold to determine whether the point should be root cause. The range of this threshold should be in [0,1]. If the point's delta is equal to or larger than rootCauseThreshold multiplied by anomaly dimension point's delta, this point is treated as a root cause. Different threshold will turn out different results. Users can choose the delta according to their data and requirments. + /// + internal double RootCauseThreshold; + public RootCauseOptions() + { + Beta = 0.3; + RootCauseThreshold = 0.95; + } + + public RootCauseOptions(double beta, double rootCauseThreshold) + { + Beta = beta; + RootCauseThreshold = rootCauseThreshold; + } + } + internal class RootCauseScore { internal double Surprise; diff --git a/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs b/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs index 3b2b311d2a..f3ce5ea8c1 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseLocalizationType.cs @@ -11,7 +11,7 @@ namespace Microsoft.ML.TimeSeries public sealed class RootCause { /// - /// A List for root cause item. Instance of the item should be + /// A List for root cause item. Instance of the item should be . /// public List Items { get; set; } public RootCause() @@ -23,12 +23,12 @@ public RootCause() public sealed class RootCauseLocalizationInput { /// - /// When the anomaly incident occurs + /// When the anomaly incident occurs. /// public DateTime AnomalyTimestamp { get; set; } /// - /// Point with the anomaly dimension must exist in the slice list at the anomaly timestamp, or the libary will not calculate the root cause + /// Point with the anomaly dimension must exist in the slice list at the anomaly timestamp, or the libary will not calculate the root cause. /// public Dictionary AnomalyDimension { get; set; } @@ -38,7 +38,7 @@ public sealed class RootCauseLocalizationInput public List Slices { get; set; } /// - /// The aggregated type, the type should be + /// The aggregated type, the type should be . /// public AggregateType AggregateType { get; set; } @@ -119,11 +119,11 @@ public sealed class RootCauseItem : IEquatable /// public List Path; /// - /// The dimension for the detected root cause point + /// The dimension for the detected root cause point. /// public Dictionary Dimension; /// - /// The direction for the detected root cause point, should be + /// The direction for the detected root cause point, should be . /// public AnomalyDirection Direction; @@ -158,7 +158,7 @@ public bool Equals(RootCauseItem other) public sealed class MetricSlice { /// - /// Timestamp for the point list + /// Timestamp for the point list. /// public DateTime TimeStamp { get; set; } /// @@ -178,15 +178,15 @@ public MetricSlice() { } public sealed class TimeSeriesPoint : IEquatable { /// - /// Value of a time series point + /// Value of a time series point. /// public double Value { get; set; } /// - /// Forecasted value for the time series point + /// Forecasted value for the time series point. /// public double ExpectedValue { get; set; } /// - /// Whether the point is an anomaly point + /// Whether the point is an anomaly point. /// public bool IsAnomaly { get; set; } /// @@ -194,7 +194,7 @@ public sealed class TimeSeriesPoint : IEquatable /// public Dictionary Dimension { get; set; } /// - /// Difference between value and expected value + /// Difference between value and expected value. /// public double Delta { get; set; } From 8e5784e85897e1a86431dcdb50839865402d4031 Mon Sep 17 00:00:00 2001 From: suxi-ms Date: Fri, 19 Jun 2020 10:29:03 +0800 Subject: [PATCH 36/36] revert option changes --- .../ExtensionsCatalog.cs | 16 +++++++----- .../RootCauseAnalyzer.cs | 26 +++---------------- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs index 672ede75ba..9c302668b9 100644 --- a/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.TimeSeries/ExtensionsCatalog.cs @@ -183,7 +183,12 @@ public static IDataView DetectEntireAnomalyBySrCnn(this AnomalyDetectionCatalog /// /// The anomaly detection catalog. /// Root cause's input. The data is an instance of . - /// An instance of of to indicate root cause parameters. + /// Beta is a weight parameter for user to choose. + /// It is used when score is calculated for each root cause item. The range of beta should be in [0,1]. + /// For a larger beta, root cause items which have a large difference between value and expected value will get a high score. + /// For a small beta, root cause items which have a high relative change will get a low score. + /// A threshold to determine whether the point should be root cause. The range of this threshold should be in [0,1]. + /// If the point's delta is equal to or larger than rootCauseThreshold multiplied by anomaly dimension point's delta, this point is treated as a root cause. Different threshold will turn out different results. Users can choose the delta according to their data and requirments. /// /// /// /// /// - public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, RootCauseLocalizationInput src, RootCauseOptions options = null) + public static RootCause LocalizeRootCause(this AnomalyDetectionCatalog catalog, RootCauseLocalizationInput src, double beta = 0.3, double rootCauseThreshold = 0.95) { - options = options ?? new RootCauseOptions(); IHostEnvironment host = CatalogUtils.GetEnvironment(catalog); //check the root cause input CheckRootCauseInput(host, src); //check parameters - host.CheckUserArg(options.Beta >= 0 && options.Beta <= 1, nameof(options.Beta), "Must be in [0,1]"); - host.CheckUserArg(options.RootCauseThreshold >= 0 && options.RootCauseThreshold <= 1, nameof(options.Beta), "Must be in [0,1]"); + host.CheckUserArg(beta >= 0 && beta <= 1, nameof(beta), "Must be in [0,1]"); + host.CheckUserArg(rootCauseThreshold >= 0 && rootCauseThreshold <= 1, nameof(beta), "Must be in [0,1]"); //find out the root cause - RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, options); + RootCauseAnalyzer analyzer = new RootCauseAnalyzer(src, beta, rootCauseThreshold); RootCause dst = analyzer.Analyze(); return dst; } diff --git a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs index f2c70225a6..bc18c0b248 100644 --- a/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs +++ b/src/Microsoft.ML.TimeSeries/RootCauseAnalyzer.cs @@ -19,11 +19,11 @@ public class RootCauseAnalyzer private double _beta; private double _rootCauseThreshold; - public RootCauseAnalyzer(RootCauseLocalizationInput src, RootCauseOptions options) + public RootCauseAnalyzer(RootCauseLocalizationInput src, double beta, double rootCauseThreshold) { _src = src; - _beta = options.Beta; - _rootCauseThreshold = options.RootCauseThreshold; + _beta = beta; + _rootCauseThreshold = rootCauseThreshold; } public RootCause Analyze() @@ -755,26 +755,6 @@ public int CompareTo(object obj) } } - public class RootCauseOptions - { - internal double Beta; - /// - /// A threshold to determine whether the point should be root cause. The range of this threshold should be in [0,1]. If the point's delta is equal to or larger than rootCauseThreshold multiplied by anomaly dimension point's delta, this point is treated as a root cause. Different threshold will turn out different results. Users can choose the delta according to their data and requirments. - /// - internal double RootCauseThreshold; - public RootCauseOptions() - { - Beta = 0.3; - RootCauseThreshold = 0.95; - } - - public RootCauseOptions(double beta, double rootCauseThreshold) - { - Beta = beta; - RootCauseThreshold = rootCauseThreshold; - } - } - internal class RootCauseScore { internal double Surprise;