Implement YamlPath for YamlDotNet (#509)

AppInspector.RulesEngine/AbstractRuleSet.cs

@@ -203,7 +203,7 @@ public IEnumerable<ConvertedOatRule> GetUniversalRules()
             var modifiers = pattern.Modifiers?.ToList() ?? new List<string>();
             if (pattern.PatternType is PatternType.String or PatternType.Substring)
                 return new OatSubstringIndexClause(scopes, useWordBoundaries: pattern.PatternType == PatternType.String,
-                    xPaths: pattern.XPaths, jsonPaths: pattern.JsonPaths)
+                    xPaths: pattern.XPaths, jsonPaths: pattern.JsonPaths, yamlPaths:pattern.YamlPaths)
                 {
                     Label = clauseNumber.ToString(CultureInfo
                         .InvariantCulture), //important to pattern index identification
@@ -212,7 +212,7 @@ public IEnumerable<ConvertedOatRule> GetUniversalRules()
                     Arguments = pattern.Modifiers?.ToList() ?? new List<string>()
                 };
             if (pattern.PatternType == PatternType.Regex)
-                return new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths)
+                return new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths, pattern.YamlPaths)
                 {
                     Label = clauseNumber.ToString(CultureInfo
                         .InvariantCulture), //important to pattern index identification

AppInspector.RulesEngine/AppInspector.RulesEngine.csproj

@@ -30,31 +30,33 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="JsonCons.JsonPath" Version="1.1.0"/>
-        <PackageReference Include="Microsoft.CST.OAT" Version="1.2.25"/>
-        <PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.1.11"/>
-        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1"/>
-        <PackageReference Include="Newtonsoft.Json" Version="13.0.1"/>
+        <PackageReference Include="JsonCons.JsonPath" Version="1.1.0" />
+        <PackageReference Include="Microsoft.CST.OAT" Version="1.2.25" />
+        <PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.1.11" />
+        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
+        <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+        <PackageReference Include="YamlDotNet" Version="12.0.1" />
 
     </ItemGroup>
 
     <ItemGroup>
-        <Content Remove="Resources\comments.json"/>
-        <Content Remove="Resources\languages.json"/>
+        <Content Remove="Resources\comments.json" />
+        <Content Remove="Resources\languages.json" />
     </ItemGroup>
 
     <ItemGroup>
-        <EmbeddedResource Include="Resources\comments.json"/>
-        <EmbeddedResource Include="Resources\languages.json"/>
+        <EmbeddedResource Include="Resources\comments.json" />
+        <EmbeddedResource Include="Resources\languages.json" />
     </ItemGroup>
 
     <ItemGroup>
-        <ProjectReference Include="..\AppInspector.Common\AppInspector.Common.csproj"/>
+        <ProjectReference Include="..\AppInspector.Common\AppInspector.Common.csproj" />
+        <ProjectReference Include="..\AppInspector.YamlPath\AppInspector.YamlPath.csproj" />
     </ItemGroup>
 
     <ItemGroup>
-        <None Include="..\.editorconfig" Link=".editorconfig"/>
-        <None Include="..\LICENSE.txt" Pack="true" PackagePath=""/>
-        <None Include="..\icon-128.png" Pack="true" PackagePath=""/>
+        <None Include="..\.editorconfig" Link=".editorconfig" />
+        <None Include="..\LICENSE.txt" Pack="true" PackagePath="" />
+        <None Include="..\icon-128.png" Pack="true" PackagePath="" />
     </ItemGroup>
 </Project>

AppInspector.RulesEngine/OatExtensions/OatRegexWithIndexClause.cs

@@ -1,23 +1,26 @@
 // Copyright (C) Microsoft. All rights reserved. Licensed under the MIT License.
 
+using System.Collections.Generic;
 using Microsoft.CST.OAT;
 
 namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions;
 
 public class OatRegexWithIndexClause : Clause
 {
     public OatRegexWithIndexClause(PatternScope[] scopes, string? field = null, string[]? xPaths = null,
-        string[]? jsonPaths = null) : base(Operation.Custom, field)
+        string[]? jsonPaths = null, string[]? ymlPaths = null) : base(Operation.Custom, field)
     {
         Scopes = scopes;
         CustomOperation = "RegexWithIndex";
         XPaths = xPaths;
         JsonPaths = jsonPaths;
+        YmlPaths = ymlPaths;
     }
 
     public string[]? JsonPaths { get; }
 
     public string[]? XPaths { get; }
 
     public PatternScope[] Scopes { get; }
+    public string[]? YmlPaths { get; }
 }

AppInspector.RulesEngine/OatExtensions/OatRegexWithIndexOperation.cs

@@ -104,8 +104,18 @@ private static IEnumerable<Violation> RegexWithIndexValidationDelegate(CST.OAT.R
                                 foreach (var target in targets)
                                     outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, target.Item2));
                             }
+
+                        if (src.YmlPaths is not null)
+                            foreach (var ymlPath in src.YmlPaths)
+                            {
+                                var targets = tc.GetStringFromYmlPath(ymlPath);
+                                foreach (var target in targets)
+                                {
+                                    outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, target.Item2));
+                                }
+                            }
 
-                        if (src.JsonPaths is null && src.XPaths is null)
+                        if (src.JsonPaths is null && src.XPaths is null && src.YmlPaths is null)
                         {
                             if (subBoundary is not null)
                                 outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, subBoundary));

AppInspector.RulesEngine/OatExtensions/OatSubstringIndexClause.cs

@@ -7,15 +7,18 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions;
 public class OatSubstringIndexClause : Clause
 {
     public OatSubstringIndexClause(PatternScope[] scopes, string? field = null, bool useWordBoundaries = false,
-        string[]? xPaths = null, string[]? jsonPaths = null) : base(Operation.Custom, field)
+        string[]? xPaths = null, string[]? jsonPaths = null, string[]? yamlPaths = null) : base(Operation.Custom, field)
     {
         Scopes = scopes;
         CustomOperation = "SubstringIndex";
         UseWordBoundaries = useWordBoundaries;
         XPaths = xPaths;
         JsonPaths = jsonPaths;
+        YmlPaths = yamlPaths;
     }
 
+    public string[]? YmlPaths { get; }
+
     public string[]? JsonPaths { get; }
 
     public string[]? XPaths { get; }

AppInspector.RulesEngine/OatExtensions/OatSubstringIndexOperation.cs

@@ -98,8 +98,21 @@ public static IEnumerable<Violation> SubstringIndexValidationDelegate(CST.OAT.Ru
                                 }
                             }
                         }
+                    if (src.YmlPaths is not null)
+                        foreach (var ymlPath in src.YmlPaths)
+                        {
+                            var targets = tc.GetStringFromYmlPath(ymlPath);
+                            foreach (var target in targets)
+                            {
+                                var matches = GetMatches(target.Item1, stringList[i], comparisonType, tc, src);
+                                foreach (var match in matches)
+                                {
+                                    outmatches.Add((i, match));
+                                }
+                            }
+                        }
 
-                    if (src.JsonPaths is null && src.XPaths is null)
+                    if (src.JsonPaths is null && src.XPaths is null && src.YmlPaths is null)
                     {
                         // If state 2 is a boundary, restrict the text provided to check to match the boundary
                         if (state2 is Boundary boundary)

AppInspector.RulesEngine/OatExtensions/WithinOperation.cs

@@ -176,15 +176,17 @@ public IEnumerable<Violation> WithinValidationDelegate(CST.OAT.Rule rule, Clause
 
                 if (wc.SubClause is OatRegexWithIndexClause oatRegexWithIndexClause)
                     if ((oatRegexWithIndexClause.JsonPaths?.Any() ?? false) ||
-                        (oatRegexWithIndexClause.XPaths?.Any() ?? false))
+                        (oatRegexWithIndexClause.XPaths?.Any() ?? false)||
+                        (oatRegexWithIndexClause.YmlPaths?.Any() ?? false))
                         if (wc.FindingOnly || wc.SameLineOnly || wc.FindingRegion || wc.OnlyAfter || wc.OnlyBefore)
-                            yield return new Violation("When providing JSONPaths or XPaths must use same-file region.",
+                            yield return new Violation("When providing JSONPaths, YMLPaths or XPaths must use same-file region.",
                                 rule, clause);
                 if (wc.SubClause is OatSubstringIndexClause oatSubstringIndexClause)
                     if ((oatSubstringIndexClause.JsonPaths?.Any() ?? false) ||
-                        (oatSubstringIndexClause.XPaths?.Any() ?? false))
+                        (oatSubstringIndexClause.XPaths?.Any() ?? false) || 
+                        (oatSubstringIndexClause.YmlPaths?.Any() ?? false))
                         if (wc.FindingOnly || wc.SameLineOnly || wc.FindingRegion || wc.OnlyAfter || wc.OnlyBefore)
-                            yield return new Violation("When providing JSONPaths or XPaths must use same-file region.",
+                            yield return new Violation("When providing JSONPaths, YMLPaths or XPaths must use same-file region.",
                                 rule, clause);
             }
         }

AppInspector.RulesEngine/SearchPattern.cs

@@ -42,4 +42,11 @@ public class SearchPattern
     /// </summary>
     [JsonProperty(PropertyName = "jsonpaths")]
     public string[]? JsonPaths { get; set; }
+
+    /// <summary>
+    ///     If set, attempt to parse the file as YML and if that is possible,
+    ///     before running the pattern, select down to the JsonPath provided
+    /// </summary>
+    [JsonProperty(PropertyName = "ymlpaths")]
+    public string[]? YamlPaths { get; set; }
 }

AppInspector.RulesEngine/TextContainer.cs

@@ -9,8 +9,10 @@
 using System.Text.Json;
 using System.Xml.XPath;
 using JsonCons.JsonPath;
+using Microsoft.ApplicationInspector.ExtensionMethods;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
+using YamlDotNet.RepresentationModel;
 
 namespace Microsoft.ApplicationInspector.RulesEngine;
 
@@ -30,6 +32,8 @@ public class TextContainer
 
     private bool _triedToConstructXPathDocument;
     private XPathDocument? _xmlDoc;
+    private bool _triedToConstructYmlDocument;
+    private YamlStream _ymlDocument;
 
     /// <summary>
     ///     Creates new instance
@@ -316,4 +320,30 @@ public bool ScopeMatch(IEnumerable<PatternScope> scopes, Boundary boundary)
         return (!isInComment && scopes.Contains(PatternScope.Code)) ||
                (isInComment && scopes.Contains(PatternScope.Comment));
     }
+
+    internal IEnumerable<(string, Boundary)> GetStringFromYmlPath(string Path)
+    {
+        if (!_triedToConstructYmlDocument)
+            try
+            {
+                _triedToConstructYmlDocument = true;
+                _ymlDocument = new YamlStream();
+                _ymlDocument.Load(new StringReader(FullContent));
+            }
+            catch (Exception e)
+            {
+                _logger.LogError("Failed to parse as a YML document: {0}", e.Message);
+                _ymlDocument = null;
+            }
+
+        if (_ymlDocument is not null)
+        {
+            var matches = _ymlDocument.Documents[0].RootNode.Query(Path);
+            foreach (var match in matches)
+            {
+                yield return (match.ToString(),
+                    new Boundary() { Index = match.Start.Index, Length = match.End.Index - match.Start.Index });
+            }
+        }
+    }
 }

AppInspector.Tests/RuleProcessor/XmlAndJsonTests.cs

@@ -9,7 +9,7 @@ namespace AppInspector.Tests.RuleProcessor;
 public class XmlAndJsonTests
 {
     private readonly Microsoft.ApplicationInspector.RulesEngine.Languages _languages = new();
-    
+
     private const string jsonAndXmlStringRule = @"[
         {
             ""id"": ""SA000005"",
@@ -149,7 +149,7 @@ public class XmlAndJsonTests
       </book>  
   </bookstore>
 ";
-    
+
     [DataRow(jsonStringRule)]
     [DataRow(jsonAndXmlStringRule)]
     [DataTestMethod]
@@ -189,7 +189,120 @@ public void XmlStringRule(string rule)
             Assert.Fail();
         }
     }
-
+
+    [TestMethod]
+    public void TestYml()
+    {
+        var content =@"hash_name:
+  a_key: 0
+  b_key: 1
+  c_key: 2
+  d_key: 3
+  e_key: 4";
+        var ruleThatWontFind = @"[{
+    ""name"": ""YamlPathValidate"",
+    ""id"": ""YmlPath"",
+    ""description"": ""Yaml Path Validation"",
+    ""tags"": [
+      ""Code.Java.17""
+    ],
+    ""severity"": ""critical"",
+    ""patterns"": [
+      {
+        ""pattern"": ""3"",
+        ""ymlpaths"" : [""/hash_name/b_key""],
+        ""type"": ""string"",
+        ""scopes"": [
+          ""code""
+        ],
+        ""modifiers"": [
+          ""i""
+        ],
+        ""confidence"": ""high""
+      }
+    ]
+  }]";
+        var ruleWithRegex = @"[{
+    ""name"": ""YamlPathValidate"",
+    ""id"": ""YmlPath"",
+    ""description"": ""Yaml Path Validation"",
+    ""tags"": [
+      ""Code.Java.17""
+    ],
+    ""severity"": ""critical"",
+    ""patterns"": [
+      {
+        ""pattern"": ""0"",
+        ""ymlpaths"" : [""/hash_name/a_key""],
+        ""type"": ""regex"",
+        ""scopes"": [
+          ""code""
+        ],
+        ""modifiers"": [
+          ""i""
+        ],
+        ""confidence"": ""high""
+      }
+    ]
+  }]";
+        var rule = @"[{
+    ""name"": ""YamlPathValidate"",
+    ""id"": ""YmlPath"",
+    ""description"": ""Yaml Path Validation"",
+    ""tags"": [
+      ""Code.Java.17""
+    ],
+    ""severity"": ""critical"",
+    ""patterns"": [
+      {
+        ""pattern"": ""0"",
+        ""ymlpaths"" : [""/hash_name/a_key""],
+        ""type"": ""string"",
+        ""scopes"": [
+          ""code""
+        ],
+        ""modifiers"": [
+          ""i""
+        ],
+        ""confidence"": ""high""
+      }
+    ]
+  }]";
+        // This rule should find one match
+        RuleSet rules = new();
+        var originalSource = "TestRules";
+        rules.AddString(rule, originalSource);
+        var analyzer = new Microsoft.ApplicationInspector.RulesEngine.RuleProcessor(rules,
+            new RuleProcessorOptions { Parallel = false, AllowAllTagsInBuildFiles = true });
+        if (_languages.FromFileNameOut("test.yml", out var info))
+        {
+            var matches = analyzer.AnalyzeFile(content, new FileEntry("test.yml", new MemoryStream()), info);
+            Assert.AreEqual(1, matches.Count);
+        }
+        rules = new();
+
+        // This rule intentionally does not find a match
+        rules.AddString(ruleThatWontFind, originalSource);
+        analyzer = new Microsoft.ApplicationInspector.RulesEngine.RuleProcessor(rules,
+            new RuleProcessorOptions { Parallel = false, AllowAllTagsInBuildFiles = true });
+        if (_languages.FromFileNameOut("test.yml", out var info2))
+        {
+            var matches = analyzer.AnalyzeFile(content, new FileEntry("test.yml", new MemoryStream()), info2);
+            Assert.AreEqual(0, matches.Count);
+        }
+        rules = new();
+
+        // This is the same rule as the first but with the regex operation
+        rules.AddString(ruleWithRegex, originalSource);
+        analyzer = new Microsoft.ApplicationInspector.RulesEngine.RuleProcessor(rules,
+            new RuleProcessorOptions { Parallel = false, AllowAllTagsInBuildFiles = true });
+        if (_languages.FromFileNameOut("test.yml", out var info3))
+        {
+            var matches = analyzer.AnalyzeFile(content, new FileEntry("test.yml", new MemoryStream()), info3);
+            Assert.AreEqual(1, matches.Count);
+        }
+    }
+
     [TestMethod]
     public void TestXmlWithAndWithoutNamespace()
     {