Strengthen ability to drive analysis via XML config files. (#2643)

* Strengthen ability to drive analysis via XML config files.

* Update release notes.

ReleaseHistory.md

@@ -1,5 +1,7 @@
 # SARIF Package Release History (SDK, Driver, Converters, and Multitool)
 ## **v4.2.0** UNRELEASED
+[#2643](https://github.com/microsoft/sarif-sdk/pull/2643)
+* BRK: Rename `Errors.LogExceptionCreatingLogFile` to `Errors.LogExceptionCreatingOutputFile` to reflect its general purpose. [#2643](https://github.com/microsoft/sarif-sdk/pull/2643)
 * BRK: Add `IAnalysisContext.FileRegionsCache` property. Used for data sharing across analysis phases. [#2642](https://github.com/microsoft/sarif-sdk/pull/2642)
 * BRK: Remove `FileRegionsCache.Instance` singleton object. Analysis should always prefer context file region context instead. [#2642](https://github.com/microsoft/sarif-sdk/pull/2642)
 * BRK: `fileRegionsCache` parameter is now required for the `InsertOptionalDataVisitor`. [#2642](https://github.com/microsoft/sarif-sdk/pull/2642)
@@ -10,6 +12,8 @@
 * BUG: Generate `IAnalysisLogger.AnalyzingTarget` callbacks from `MulthreadedAnalyzeCommandBase`. [#2637](https://github.com/microsoft/sarif-sdk/pull/2637)
 * BUG: Persist `fileRegionsCache` parameter in `SarifLogger` to support retrieving hash data. [#2639](https://github.com/microsoft/sarif-sdk/pull/2639)
 * BUG: Allow override of `FailureLevels` and `ResultKinds` in context objects. [#2639](https://github.com/microsoft/sarif-sdk/pull/2639)
+* NEW: Add `DefaultTraces.ResultsSummary` property that drives naive results summary in console logger. [#2643](https://github.com/microsoft/sarif-sdk/pull/2643)
+* NEW: Prove `AnalyzeContextBase.Inline` helper. [#2643](https://github.com/microsoft/sarif-sdk/pull/2643)
 * NEW: `SarifLogger.FileRegionsCache` property added (to support sharing this instance with context and other classes). [#2642](https://github.com/microsoft/sarif-sdk/pull/2642)
 * NEW: `MultithreadedAnalyzeCommandBase.Tool` is now public to support in-memory analysis (and logging) of targets. [#2639](https://github.com/microsoft/sarif-sdk/pull/2639)
 * NEW: Add `DefaultTraces.TargetsScanned` which is used by `ConsoleLogger` to emit target start and stop analysis messages. [#2637](https://github.com/microsoft/sarif-sdk/pull/2637)

src/Sarif.Driver/ResultsSummaryLogger.cs

@@ -0,0 +1,95 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Linq;
+
+namespace Microsoft.CodeAnalysis.Sarif.Driver
+{
+    internal class ResultsSummaryLogger : IAnalysisLogger, IDisposable
+    {
+        public FileRegionsCache FileRegionsCache { get; set; }
+
+        public void AnalysisStarted()
+        {
+        }
+
+        public void AnalysisStopped(RuntimeConditions runtimeConditions)
+        {
+        }
+
+        public void AnalyzingTarget(IAnalysisContext context)
+        {
+        }
+
+        private Dictionary<string, Tuple<int, int, int>> rulesSummary;
+
+        public void Log(ReportingDescriptor rule, Result result, int? extensionIndex = null)
+        {
+            string key = $"{rule.Id}.{rule.Name}";
+            rulesSummary ??= new Dictionary<string, Tuple<int, int, int>>();
+            rulesSummary.TryGetValue(key, out Tuple<int, int, int> tuple);
+            tuple ??= new Tuple<int, int, int>(0, 0, 0);
+
+            switch (result.Level)
+            {
+                case FailureLevel.Error:
+                {
+                    tuple = new Tuple<int, int, int>(tuple.Item1 + 1, tuple.Item2, tuple.Item3);
+                    break;
+                }
+                case FailureLevel.Warning:
+                {
+                    tuple = new Tuple<int, int, int>(tuple.Item1 + 1, tuple.Item2, tuple.Item3);
+                    break;
+                }
+                case FailureLevel.Note:
+                {
+                    tuple = new Tuple<int, int, int>(tuple.Item1 + 1, tuple.Item2, tuple.Item3);
+                    break;
+                }
+            }
+
+            rulesSummary[key] = tuple;
+        }
+
+        public void LogConfigurationNotification(Notification notification)
+        {
+        }
+
+        public void LogToolNotification(Notification notification, ReportingDescriptor associatedRule = null)
+        {
+        }
+
+        public void TargetAnalyzed(IAnalysisContext context)
+        {
+        }
+
+        public void Dispose()
+        {
+            if (this.rulesSummary == null) { return; }
+
+            Console.WriteLine();
+
+            var aggregatedCounts = new Dictionary<string, long>();
+            foreach (string ruleIdAndName in this.rulesSummary.Keys)
+            {
+                Tuple<int, int, int> tuple = this.rulesSummary[ruleIdAndName];
+                long count = tuple.Item1 + tuple.Item2 + tuple.Item3;
+                string line = $"{ruleIdAndName} : {count}";
+                aggregatedCounts[line] = count;
+            }
+
+            IOrderedEnumerable<KeyValuePair<string, long>> sortedResults = from entry in aggregatedCounts orderby entry.Value descending select entry;
+            foreach (KeyValuePair<string, long> line in sortedResults)
+            {
+                Console.WriteLine(line.Key);
+            }
+
+            Console.WriteLine();
+        }
+    }
+}

src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs

@@ -28,7 +28,7 @@ public abstract class AnalyzeOptionsBase : CommonOptionsBase
             'r',
             "recurse",
             HelpText = "Recurse into subdirectories when evaluating file specifier arguments.")]
-        public bool Recurse { get; set; }
+        public bool? Recurse { get; set; }
 
         [Option(
             'c',
@@ -42,20 +42,6 @@ public abstract class AnalyzeOptionsBase : CommonOptionsBase
             HelpText = "Suppress all console output (except for catastrophic tool runtime or configuration errors).")]
         public bool? Quiet { get; set; }
 
-        [Option(
-            's',
-            "statistics",
-            HelpText = "Generate timing and other statistics for analysis session.")]
-        [Obsolete()]
-        public bool Statistics { get; set; }
-
-        [Option(
-            'h',
-            "hashes",
-            HelpText = "Output MD5, SHA1, and SHA-256 hash of analysis targets when emitting SARIF reports.")]
-        [Obsolete("Use --insert instead, passing 'Hashes' along with any other references to data to be inserted.")]
-        public bool ComputeFileHashes { get; set; }
-
         [Option(
             'e',
             "environment",
@@ -133,7 +119,6 @@ public IEnumerable<ResultKind> Kind
             set => this.kind = value?.Count() > 0 ? value : null;
         }
 
-
         public ResultKindSet ResultKinds => Kind != null ? new ResultKindSet(Kind) : BaseLogger.Fail;
 
         [Option(

src/Sarif.Driver/Sdk/CommonOptionsBase.cs

@@ -24,7 +24,6 @@ public class CommonOptionsBase : PropertiesDictionary
 
         public bool ForceOverwrite => OutputFileOptions.ToFlags().HasFlag(FilePersistenceOptions.ForceOverwrite);
 
-
         private HashSet<FilePersistenceOptions> _filePersistenceOptions;
 
         [Option(
@@ -46,10 +45,12 @@ internal static IEnumerable<FilePersistenceOptions> NormalizeFilePersistenceOpti
                 filePersistenceOptions.Remove(FilePersistenceOptions.Minify);
             }
 
-            if (!filePersistenceOptions.Contains(FilePersistenceOptions.Minify))
+            if (filePersistenceOptions.Count > 0 &&
+                !filePersistenceOptions.Contains(FilePersistenceOptions.Minify))
             {
                 filePersistenceOptions.Add(FilePersistenceOptions.PrettyPrint);
             }
+
             return filePersistenceOptions;
         }
 

src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs

@@ -156,7 +156,7 @@ private int Run(TContext globalContext)
             globalContext.FileSystem ??= FileSystem;
             globalContext = ValidateContext(globalContext);
             disposableLogger = globalContext.Logger as IDisposable;
-            InitializeOutputFile(globalContext);
+            InitializeOutputs(globalContext);
 
             // 1. Instantiate skimmers. We need to do this before initializing
             //    the output file so that we can preconstruct the tool 
@@ -188,6 +188,8 @@ private int Run(TContext globalContext)
             globalContext.Logger.AnalysisStopped(globalContext.RuntimeErrors);
             disposableLogger?.Dispose();
 
+            // Note that we don't clear the logger here. That is because the console
+            // logger and any other custom loggers can still be useful for these operations.
             if ((globalContext.RuntimeErrors & ~RuntimeConditions.Nonfatal) == RuntimeConditions.None)
             {
                 ProcessBaseline(globalContext);
@@ -217,36 +219,25 @@ public virtual TContext InitializeContextFromOptions(TOptions options, ref TCont
             // XML but to override specific settings within it via options.
             context = InitializeConfiguration(options.ConfigurationFilePath, context);
 
-            // TBD: observe that unless/until all options are nullable, that we
-            // will always clobber loaded context data when processing options.
-            // The 'Quiet' property shows the model, use of a nullable type.
-            // We need to convert all remaining options to nullable. Note that
-            // there's still a problem: we can't use the absence of a command-line
-            // boolean argument as positive evidence that a context representation
-            // should be overridden. i.e., say the context file specifies 'Quiet',
-            // there's no way that the options can override this because a value
-            // of false is implied by the absence of an explicit command-line arg.
-            // One solution is remove all boolean args, as we did with --force, 
-            // in preference of enum-driven settings.
             context.Quiet = options.Quiet != null ? options.Quiet.Value : context.Quiet;
-            context.Recurse = options.Recurse;
-            context.Threads = options.Threads > 0 ? options.Threads : Environment.ProcessorCount;
-            context.PostUri = options.PostUri;
-            context.AutomationId = options.AutomationId;
-            context.OutputFilePath = options.OutputFilePath;
-            context.AutomationGuid = options.AutomationGuid;
-            context.BaselineFilePath = options.BaselineFilePath;
+            context.Recurse = options.Recurse != null ? options.Recurse.Value : context.Recurse;
+            context.Threads = options.Threads > 0 ? options.Threads : context.Threads;
+            context.PostUri = options.PostUri != null ? options.PostUri : context.PostUri;
+            context.AutomationId = options.AutomationId != null ? options.AutomationId : context.AutomationId;
+            context.OutputFilePath = options.OutputFilePath != null ? options.OutputFilePath : context.OutputFilePath;
+            context.AutomationGuid = options != default ? options.AutomationGuid : context.AutomationGuid;
+            context.BaselineFilePath = options.BaselineFilePath != null ? options.BaselineFilePath : context.BaselineFilePath;
             context.Traces = options.Trace != null ? InitializeStringSet(options.Trace) : context.Traces;
-            context.DataToInsert = options.DataToInsert.ToFlags();
-            context.DataToRemove = options.DataToRemove.ToFlags();
+            context.DataToInsert = options.DataToInsert?.Any() == true ? options.DataToInsert.ToFlags() : context.DataToInsert;
+            context.DataToRemove = options.DataToRemove?.Any() == true ? options.DataToRemove.ToFlags() : context.DataToRemove;
             context.ResultKinds = options.Kind != null ? options.ResultKinds : context.ResultKinds;
             context.FailureLevels = options.Level != null ? options.FailureLevels : context.FailureLevels;
-            context.OutputFileOptions = options.OutputFileOptions.ToFlags();
+            context.OutputFileOptions = options.OutputFileOptions?.Any() == true ? options.OutputFileOptions.ToFlags() : context.OutputFileOptions;
             context.MaxFileSizeInKilobytes = options.MaxFileSizeInKilobytes != null ? options.MaxFileSizeInKilobytes.Value : context.MaxFileSizeInKilobytes;
-            context.PluginFilePaths = options.PluginFilePaths?.ToImmutableHashSet();
-            context.InsertProperties = InitializeStringSet(options.InsertProperties);
-            context.TargetFileSpecifiers = options.TargetFileSpecifiers != null ? InitializeStringSet(options.TargetFileSpecifiers) : context.TargetFileSpecifiers;
-            context.InvocationPropertiesToLog = InitializeStringSet(options.InvocationPropertiesToLog);
+            context.PluginFilePaths = options.PluginFilePaths?.Any() == true ? options.PluginFilePaths?.ToImmutableHashSet() : context.PluginFilePaths;
+            context.InsertProperties = options.InsertProperties?.Any() == true ? InitializeStringSet(options.InsertProperties) : context.InsertProperties;
+            context.TargetFileSpecifiers = options.TargetFileSpecifiers?.Any() == true ? InitializeStringSet(options.TargetFileSpecifiers) : context.TargetFileSpecifiers;
+            context.InvocationPropertiesToLog = options.InvocationPropertiesToLog?.Any() == true ? InitializeStringSet(options.InvocationPropertiesToLog) : context.InvocationPropertiesToLog;
 
             if (context.TargetsProvider == null)
             {
@@ -348,30 +339,30 @@ private static ISet<string> InitializeStringSet(IEnumerable<string> strings)
 
             // 1: First we initiate an asynchronous operation to locate disk files for
             // analysis, as specified in analysis configuration (file names, wildcards).
-            Task<bool> enumerateFilesOnDisk = Task.Run(() => EnumerateFilesOnDiskAsync(globalContext));
+            Task<bool> enumerateTargets = Task.Run(() => EnumerateTargetsAsync(globalContext));
 
             // 2: A dedicated set of threads pull scan targets and analyze them.
             //    On completing a scan, the thread writes the index of the 
             //    scanned item to a channel that drives logging.
-            var workers = new Task[globalContext.Threads];
+            var scanWorkers = new Task[globalContext.Threads];
             for (int i = 0; i < globalContext.Threads; i++)
             {
-                workers[i] = Task.Run(() => ScanTargetsAsync(globalContext, skimmers, disabledSkimmers));
+                scanWorkers[i] = Task.Run(() => ScanTargetsAsync(globalContext, skimmers, disabledSkimmers));
             }
 
             // 3: A single-threaded consumer watches for completed scans
             //    and logs results, if any. This operation is single-threaded
             //    to ensure determinism in log output. i.e., any scan of the
             //    same targets using the same production code should produce
             //    a log file that is byte-for-byte identical to previous log.
-            Task logScanResults = Task.Run(() => LogScanResultsAsync(globalContext));
+            Task logResults = Task.Run(() => LogResultsAsync(globalContext));
 
-            Task.WhenAll(workers)
+            Task.WhenAll(scanWorkers)
                 .ContinueWith(_ => _resultsWritingChannel.Writer.Complete())
                 .Wait();
 
-            enumerateFilesOnDisk.Wait();
-            logScanResults.Wait();
+            enumerateTargets.Wait();
+            logResults.Wait();
 
             if (_ignoredFilesCount > 0)
             {
@@ -403,7 +394,7 @@ private static ISet<string> InitializeStringSet(IEnumerable<string> strings)
             }
         }
 
-        private async Task LogScanResultsAsync(TContext globalContext)
+        private async Task LogResultsAsync(TContext globalContext)
         {
             uint currentIndex = 0;
 
@@ -522,7 +513,7 @@ private static void LogCachingLogger(TContext globalContext, CachingLogger cachi
             globalContext.Logger.TargetAnalyzed(globalContext);
         }
 
-        private async Task<bool> EnumerateFilesOnDiskAsync(TContext context)
+        private async Task<bool> EnumerateTargetsAsync(TContext context)
         {
             try
             {
@@ -757,7 +748,7 @@ protected virtual TContext InitializeConfiguration(string configurationFileName,
             return context;
         }
 
-        public virtual void InitializeOutputFile(TContext globalContext)
+        public virtual void InitializeOutputs(TContext globalContext)
         {
             string filePath = globalContext.OutputFilePath;
 
@@ -771,6 +762,11 @@ public virtual void InitializeOutputFile(TContext globalContext)
                     globalContext.Logger = aggregatingLogger;
                 }
 
+                if (globalContext.Traces.Contains(nameof(DefaultTraces.ResultsSummary)))
+                {
+                    aggregatingLogger.Loggers.Add(new ResultsSummaryLogger());
+                }
+
                 InvokeCatchingRelevantIOExceptions
                 (
                     () =>
@@ -810,7 +806,7 @@ public virtual void InitializeOutputFile(TContext globalContext)
                     },
                     (ex) =>
                     {
-                        Errors.LogExceptionCreatingLogFile(globalContext, filePath, ex);
+                        Errors.LogExceptionCreatingOutputFile(globalContext, filePath, ex);
                         ThrowExitApplicationException(ExitReason.ExceptionCreatingLogFile, ex);
                     }
                 );

src/Sarif/AnalyzeContextBase.cs

@@ -61,6 +61,7 @@ public virtual IEnumerable<IOption> GetOptions()
         public virtual RuntimeConditions RuntimeErrors { get; set; }
         public virtual bool AnalysisComplete { get; set; }
 
+        public bool Inline => OutputFileOptions.HasFlag(FilePersistenceOptions.Inline);
         public bool Minify => OutputFileOptions.HasFlag(FilePersistenceOptions.Minify);
         public bool Optimize => OutputFileOptions.HasFlag(FilePersistenceOptions.Optimize);
         public bool PrettyPrint => OutputFileOptions.HasFlag(FilePersistenceOptions.PrettyPrint);

src/Sarif/DefaultTraces.cs

@@ -25,5 +25,9 @@ public enum DefaultTraces
         /// Enables a trace message that reports progress against each scan target.
         /// </summary>
         TargetsScanned = 0x8,
+        /// <summary>
+        /// Enables a trace message that summarizes all results and notification by id and severity.
+        /// </summary>
+        ResultsSummary = 0x10,
     }
 }