Performance and misc improvements #365
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| // Copyright (C) Microsoft. All rights reserved. Licensed under the MIT License. | ||
|
|
||
| using Microsoft.CST.OAT; | ||
|
|
||
| namespace Microsoft.ApplicationInspector.RulesEngine | ||
| { | ||
| public class OATSubstringIndexClause : Clause | ||
| { | ||
| public OATSubstringIndexClause(PatternScope[] scopes, string? field = null, bool useWordBoundaries = false) : base(Operation.Custom, field) | ||
| { | ||
| Scopes = scopes; | ||
| CustomOperation = "SubstringIndex"; | ||
| UseWordBoundaries = useWordBoundaries; | ||
| } | ||
|
|
||
| public PatternScope[] Scopes { get; } | ||
|
|
||
| public bool UseWordBoundaries {get;} | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| using Microsoft.CST.OAT; | ||
| using Microsoft.CST.OAT.Operations; | ||
| using Microsoft.CST.OAT.Utils; | ||
| using Serilog; | ||
| using System; | ||
| using System.Collections.Concurrent; | ||
| using System.Collections.Generic; | ||
| using System.Globalization; | ||
| using System.Linq; | ||
| using System.Text.RegularExpressions; | ||
|
|
||
| namespace Microsoft.ApplicationInspector.RulesEngine | ||
| { | ||
| /// <summary> | ||
| /// The Custom Operation to enable identification of pattern index in result used by Application Inspector to report why a given | ||
| /// result was matched and to retrieve other pattern level meta-data | ||
| /// </summary> | ||
| public class OATSubstringIndexOperation : OatOperation | ||
| { | ||
| /// <summary> | ||
| /// Create an OatOperation given an analyzer | ||
| /// </summary> | ||
| /// <param name="analyzer">The analyzer context to work with</param> | ||
| public OATSubstringIndexOperation(Analyzer analyzer) : base(Operation.Custom, analyzer) | ||
| { | ||
| CustomOperation = "SubstringIndex"; | ||
| OperationDelegate = SubstringIndexOperationDelegate; | ||
| ValidationDelegate = SubstringIndexValidationDelegate; | ||
| } | ||
|
|
||
| public IEnumerable<Violation> SubstringIndexValidationDelegate(CST.OAT.Rule rule, Clause clause) | ||
| { | ||
| if (clause.Data?.Count == null || clause.Data?.Count == 0) | ||
| { | ||
| yield return new Violation(string.Format(Strings.Get("Err_ClauseNoData"), rule.Name, clause.Label ?? rule.Clauses.IndexOf(clause).ToString(CultureInfo.InvariantCulture)), rule, clause); | ||
|
There was a problem hiding this comment. out of curiosity, how is yield helping here? There was a problem hiding this comment. The api you implement is an enumerable so
You must implement this method as an IEnumerable to inherit from OATOperation. |
||
| } | ||
| if (clause.DictData != null && clause.DictData?.Count > 0) | ||
| { | ||
| yield return new Violation(string.Format(Strings.Get("Err_ClauseDictDataUnexpected"), rule.Name, clause.Label ?? rule.Clauses.IndexOf(clause).ToString(CultureInfo.InvariantCulture), clause.Operation.ToString()), rule, clause); | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Returns results with pattern index and Boundary as a tuple to enable retrieval of Rule pattern level meta-data like Confidence and report the | ||
| /// pattern that was responsible for the match | ||
| /// </summary> | ||
| /// <param name="clause"></param> | ||
| /// <param name="state1"></param> | ||
| /// <param name="state2"></param> | ||
| /// <param name="captures"></param> | ||
| /// <returns></returns> | ||
| public OperationResult SubstringIndexOperationDelegate(Clause clause, object? state1, object? state2, IEnumerable<ClauseCapture>? captures) | ||
| { | ||
| var comparisonType = clause.Arguments.Contains("i") ? StringComparison.InvariantCultureIgnoreCase : StringComparison.InvariantCulture; | ||
| if (state1 is TextContainer tc && clause is OATSubstringIndexClause src) | ||
| { | ||
| if (clause.Data is List<string> stringList && stringList.Any()) | ||
| { | ||
| var outmatches = new List<(int, Boundary)>();//tuple results i.e. pattern index and where | ||
|
|
||
| for (int i = 0; i < stringList.Count; i++) | ||
| { | ||
| var idx = tc.FullContent.IndexOf(stringList[i], comparisonType); | ||
| while (idx != -1) | ||
| { | ||
| bool skip = false; | ||
| if (src.UseWordBoundaries) | ||
| { | ||
| if (idx > 0 && char.IsLetterOrDigit(tc.FullContent[idx - 1])) | ||
| { | ||
| skip = true; | ||
| } | ||
| if (idx + stringList[i].Length < tc.FullContent.Length && char.IsLetterOrDigit(tc.FullContent[idx + stringList[i].Length])) | ||
| { | ||
| skip = true; | ||
| } | ||
| } | ||
| if (!skip) | ||
| { | ||
| Boundary newBoundary = new Boundary() | ||
| { | ||
| Length = stringList[i].Length, | ||
| Index = idx | ||
| }; | ||
| if (tc.ScopeMatch(src.Scopes, newBoundary)) | ||
| { | ||
| outmatches.Add((i, newBoundary)); | ||
| } | ||
| } | ||
| idx = tc.FullContent.IndexOf(stringList[i], idx + stringList[i].Length, comparisonType); | ||
| } | ||
| } | ||
|
|
||
| var result = src.Invert ? outmatches.Count == 0 : outmatches.Count > 0; | ||
| return new OperationResult(result, result && src.Capture ? new TypedClauseCapture<List<(int, Boundary)>>(clause, outmatches, state1) : null); | ||
| } | ||
| } | ||
| return new OperationResult(false, null); | ||
| } | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is OatSubstringIIndex generic enough to move into OAT: https://github.com/microsoft/OAT/tree/main/OAT/Operations?, or did I get that wrong?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
A few issues.
It's not much different than the contains operator that already exists and is much more broad in terms of the data types it can work on.
Secondly, built-in operations in OAT should not be for one specific data type (for example this just works for strings).
The main distinction is that we are capturing and returning the index of the match, a behavior that is not mirrored in any other OAT operation, and doesn't make contextual sense outside of simple string matches.
See the existing AI code and AI uses it's own regex instead of the OAT regex because OAT doesn't deal with Boundary objects, that's a AI concept.