Add WebApp.API ruleset for detecting exposed HTTP/REST APIs

AppInspector.CLI/CLICmdOptions.cs

@@ -154,7 +154,7 @@ public record CLIAnalyzeCmdOptions : CLIAnalysisSharedCommandOptions
     public bool NoFileMetadata { get; set; }
 
     [Option('A', "allow-all-tags-in-build-files", Required = false,
-        HelpText = "Allow all tags (not just Metadata tags) in files of type Build.")]
+        HelpText = "Allow non-Metadata tags from universal rules in Build files. Rules declaring applies_to or applies_to_file_regex are always eligible.")]
     public bool AllowAllTagsInBuildFiles { get; set; }
 
     [Option('M', "max-num-matches-per-tag", Required = false,

AppInspector.CLI/preferences/tagreportgroups.json

@@ -41,6 +41,11 @@
             "searchPattern": "^AI\\..*$",
             "displayName": "AI",
             "detectedIcon": "fa-solid fa-robot"
+          },
+          {
+            "searchPattern": "^WebApp\\.API\\..*$",
+            "displayName": "Exposed web API",
+            "detectedIcon": "fas fa-plug"
           }
         ]
       },

AppInspector.RulesEngine/AbstractRuleSet.cs

@@ -63,9 +63,7 @@ public IEnumerable<ConvertedOatRule> ByFilename(string input)
     /// <returns></returns>
     public IEnumerable<ConvertedOatRule> GetUniversalRules()
     {
-        return _oatRules.Where(x =>
-            (x.AppInspectorRule.FileRegexes is null || x.AppInspectorRule.FileRegexes.Count == 0) &&
-            (x.AppInspectorRule.AppliesTo is null || x.AppInspectorRule.AppliesTo.Count == 0));
+        return _oatRules.Where(x => x.AppInspectorRule.IsUniversal);
     }
 
     /// <summary>

AppInspector.RulesEngine/Resources/languages.json

@@ -108,6 +108,7 @@
   {
     "name": "kotlin",
     "extensions": [
+      ".kt",
       ".kts"
     ],
     "type": "code"

AppInspector.RulesEngine/Rule.cs

@@ -96,6 +96,14 @@ public IList<string>? FileRegexes
                 _updateCompiledFileRegex = true;
             }
         }
+
+        /// <summary>
+        ///     Gets whether the rule applies universally instead of declaring a target language or file name.
+        /// </summary>
+        [JsonIgnore]
+        public bool IsUniversal =>
+            (FileRegexes is null || FileRegexes.Count == 0) &&
+            (AppliesTo is null || AppliesTo.Count == 0);
 
         /// <summary>
         ///     Internal API to cache construction of <see cref="FileRegexes"/>

AppInspector.RulesEngine/RuleProcessor.cs

@@ -157,8 +157,10 @@ public List<MatchRecord> AnalyzeFile(TextContainer textContainer, FileEntry file
             {
                 var patternIndex = match.Item1;
                 var boundary = match.Item2;
-                //restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code
-                if (!_opts.AllowAllTagsInBuildFiles && languageInfo.Type == LanguageInfo.LangFileType.Build &&
+                // Universal rules can reach build files incidentally, so suppress their non-Metadata tags by default.
+                if (!_opts.AllowAllTagsInBuildFiles &&
+                    languageInfo.Type == LanguageInfo.LangFileType.Build &&
+                    oatRule.AppInspectorRule.IsUniversal &&
                     (oatRule.AppInspectorRule.Tags?.Any(v => !v.Contains("Metadata")) ?? false))
                 {
                     continue;
@@ -366,9 +368,10 @@ List<MatchRecord> ProcessBoundary(ClauseCapture cap)
                                 var patternIndex = match.Item1;
                                 var boundary = match.Item2;
 
-                                //restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code
+                                // Universal rules can reach build files incidentally, so suppress their non-Metadata tags by default.
                                 if (!_opts.AllowAllTagsInBuildFiles &&
                                     languageInfo.Type == LanguageInfo.LangFileType.Build &&
+                                    oatRule.AppInspectorRule.IsUniversal &&
                                     (oatRule.AppInspectorRule.Tags?.Any(v => !v.Contains("Metadata")) ?? false))
                                 {
                                     continue;

AppInspector.Tests/RuleProcessor/BuildFileRuleTests.cs

@@ -0,0 +1,90 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.ApplicationInspector.RulesEngine;
+using Microsoft.CST.RecursiveExtractor;
+using Xunit;
+
+namespace AppInspector.Tests.RuleProcessor;
+
+public class BuildFileRuleTests
+{
+    private const string BuildFileContents = "{\"value\":\"build-marker\"}";
+    private const string BuildFileName = "test.json";
+    private const string FeatureTag = "Testing.Build.Feature";
+    private const string Marker = "build-marker";
+    private readonly Microsoft.ApplicationInspector.RulesEngine.Languages _languages = new();
+
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task ExplicitBuildLanguageRuleEmitsFeatureTagByDefault(bool analyzeAsync)
+    {
+        var languageInfo = GetBuildLanguage();
+        var rule = CreateRule("BUILD000001", new[] { languageInfo.Name });
+
+        var matches = await AnalyzeAsync(rule, languageInfo, false, analyzeAsync);
+
+        var match = Assert.Single(matches);
+        Assert.Equal(FeatureTag, Assert.Single(match.Rule!.Tags!));
+    }
+
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task UniversalBuildRuleEmitsFeatureTagOnlyWhenAllowed(bool analyzeAsync)
+    {
+        var languageInfo = GetBuildLanguage();
+        var rule = CreateRule("BUILD000002");
+
+        var defaultMatches = await AnalyzeAsync(rule, languageInfo, false, analyzeAsync);
+        var allowedMatches = await AnalyzeAsync(rule, languageInfo, true, analyzeAsync);
+
+        Assert.Empty(defaultMatches);
+        var match = Assert.Single(allowedMatches);
+        Assert.Equal(FeatureTag, Assert.Single(match.Rule!.Tags!));
+    }
+
+    private static async Task<List<MatchRecord>> AnalyzeAsync(Rule rule, LanguageInfo languageInfo,
+        bool allowAllTagsInBuildFiles, bool analyzeAsync)
+    {
+        RuleSet rules = new();
+        rules.AddRule(rule);
+        Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules,
+            new RuleProcessorOptions { AllowAllTagsInBuildFiles = allowAllTagsInBuildFiles });
+        using MemoryStream stream = new(Encoding.UTF8.GetBytes(BuildFileContents));
+        FileEntry fileEntry = new(BuildFileName, stream);
+
+        return analyzeAsync
+            ? await processor.AnalyzeFileAsync(fileEntry, languageInfo)
+            : processor.AnalyzeFile(BuildFileContents, fileEntry, languageInfo);
+    }
+
+    private static Rule CreateRule(string id, string[]? appliesTo = null)
+    {
+        return new Rule
+        {
+            Id = id,
+            Name = "Build file filtering test",
+            AppliesTo = appliesTo,
+            Tags = new[] { FeatureTag },
+            Patterns = new[]
+            {
+                new SearchPattern
+                {
+                    Pattern = Marker,
+                    PatternType = PatternType.Substring,
+                    Confidence = Confidence.High
+                }
+            }
+        };
+    }
+
+    private LanguageInfo GetBuildLanguage()
+    {
+        Assert.True(_languages.FromFileNameOut(BuildFileName, out var languageInfo));
+        Assert.Equal(LanguageInfo.LangFileType.Build, languageInfo.Type);
+        return languageInfo;
+    }
+}

AppInspector.Tests/RuleProcessor/XmlAndJsonTests.cs

@@ -62,7 +62,7 @@ public void XPathVersionElementSampleBoundary()
         RuleSet rules = new();
         rules.AddString(rule, "TestRules");
         Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules,
-            new RuleProcessorOptions { AllowAllTagsInBuildFiles = true });
+            new RuleProcessorOptions());
 
         if (_languages.FromFileNameOut("pom.xml", out var info))
         {
@@ -347,7 +347,7 @@ public void XmlWithNamespaces()
         //var verification= verifier.Verify(rules);
         //Assert.Equal(true,verification.Verified);
         Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules,
-            new RuleProcessorOptions { AllowAllTagsInBuildFiles = true });
+            new RuleProcessorOptions());
         if (_languages.FromFileNameOut("AndroidManifest.xml", out var info))
         {
             var matches = processor.AnalyzeFile(@"<?xml version=""1.0"" encoding=""utf-8""?><manifest xmlns:android=""http://schemas.android.com/apk/res/android"" xmlns=""http://maven.apache.org/POM/4.0.0""><application android:debuggable='true' /></manifest>", new FileEntry("AndroidManifest.xml", new MemoryStream()), info);
@@ -390,7 +390,7 @@ public void XmlAttributeTest()
         RuleSet rules = new();
         rules.AddString(attributeRule, "JsonTestRules");
         Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules,
-            new RuleProcessorOptions { AllowAllTagsInBuildFiles = true });
+            new RuleProcessorOptions());
         if (_languages.FromFileNameOut("test.config", out var info))
         {
             var matches = processor.AnalyzeFile(attributeContent, new FileEntry("test.config", new MemoryStream()), info);

AppInspector/Commands/AnalyzeCommand.cs

@@ -61,8 +61,8 @@ public class AnalyzeOptions
     public bool SingleThread { get; set; }
 
     /// <summary>
-    ///     Treat <see cref="LanguageInfo.LangFileType.Build" /> files as if they were
-    ///     <see cref="LanguageInfo.LangFileType.Code" /> when determining if tags should apply.
+    ///     Allow universal rules to emit non-Metadata tags in <see cref="LanguageInfo.LangFileType.Build" /> files.
+    ///     Rules declaring applies_to or applies_to_file_regex are always eligible.
     /// </summary>
     public bool AllowAllTagsInBuildFiles { get; set; }