Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add baseline option for analyze command #2371

Merged
merged 4 commits into from
Jul 14, 2021
Merged

Add baseline option for analyze command #2371

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
69728b4
Add baseline option for analyze command
yongyan-gh Jul 7, 2021
5429900
Fix tests failures
yongyan-gh Jul 8, 2021
4b79d81
Test cases
yongyan-gh Jul 8, 2021
1b2ee95
Use SarifLog serialization function.
yongyan-gh Jul 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 9 additions & 1 deletion src/Sarif.Driver/DriverExtensionMethods.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,12 +160,20 @@ public static bool ValidateOutputOptions(this MultipleFilesOptionsBase options)
return valid;
}

public static bool ValidateOutputOptions(this AnalyzeOptionsBase options)
public static bool ValidateOutputOptions(this AnalyzeOptionsBase options, IAnalysisContext context)
{
bool valid = true;

valid &= !(options.Quiet && string.IsNullOrWhiteSpace(options.OutputFilePath));

// baseline process now depends on output file
valid &= !(string.IsNullOrWhiteSpace(options.OutputFilePath) && !string.IsNullOrWhiteSpace(options.BaselineSarifFile));

if (!valid)
{
context.RuntimeErrors |= RuntimeConditions.InvalidCommandLineOption;
}

return valid;
}
}
Expand Down
17 changes: 16 additions & 1 deletion src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,20 @@ public override int Run(TOptions options)

bool succeeded = (RuntimeErrors & ~RuntimeConditions.Nonfatal) == RuntimeConditions.None;

if (succeeded)
{
try
{
ProcessBaseline(_rootContext, options, FileSystem);
}
catch (Exception ex)
{
RuntimeErrors |= RuntimeConditions.ExceptionProcessingBaseline;
ExecutionException = ex;
return FAILURE;
}
}

if (options.RichReturnCode)
{
return (int)RuntimeErrors;
Expand Down Expand Up @@ -172,9 +186,10 @@ protected virtual void ValidateOptions(TContext context, TOptions analyzeOptions
succeeded &= ValidateFile(context, analyzeOptions.OutputFilePath, shouldExist: null);
succeeded &= ValidateFile(context, analyzeOptions.ConfigurationFilePath, shouldExist: true);
succeeded &= ValidateFiles(context, analyzeOptions.PluginFilePaths, shouldExist: true);
succeeded &= ValidateFile(context, analyzeOptions.BaselineSarifFile, shouldExist: true);
succeeded &= ValidateInvocationPropertiesToLog(context, analyzeOptions.InvocationPropertiesToLog);
succeeded &= ValidateOutputFileCanBeCreated(context, analyzeOptions.OutputFilePath, analyzeOptions.Force);
succeeded &= analyzeOptions.ValidateOutputOptions();
succeeded &= analyzeOptions.ValidateOutputOptions(context);

if (!succeeded)
{
Expand Down
5 changes: 5 additions & 0 deletions src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,5 +104,10 @@ public abstract class AnalyzeOptionsBase : CommonOptionsBase
Default = new ResultKind[] { ResultKind.Fail },
HelpText = "A semicolon delimited list to filter output to one or more result kinds. Valid values: Fail (for literal scan results), Pass, Review, Open, NotApplicable and Informational.")]
public IEnumerable<ResultKind> Kind { get; set; }

[Option(
"baseline",
HelpText = "A Sarif file to be used as baseline")]
public string BaselineSarifFile { get; set; }
}
}
3 changes: 2 additions & 1 deletion src/Sarif.Driver/Sdk/ExitReason.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ public enum ExitReason
UnhandledExceptionInEngine,
NoRulesLoaded,
NoValidAnalysisTargets,
InvalidCommandLineOption
InvalidCommandLineOption,
ExceptionProcessingBaseline
}
}
16 changes: 16 additions & 0 deletions src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,20 @@ public override int Run(TOptions options)

bool succeeded = (RuntimeErrors & ~RuntimeConditions.Nonfatal) == RuntimeConditions.None;

if (succeeded)
{
try
{
ProcessBaseline(_rootContext, options, FileSystem);
}
catch (Exception ex)
{
RuntimeErrors |= RuntimeConditions.ExceptionProcessingBaseline;
ExecutionException = ex;
return FAILURE;
}
}

if (options.RichReturnCode)
{
return (int)RuntimeErrors;
Expand Down Expand Up @@ -451,8 +465,10 @@ protected virtual void ValidateOptions(TOptions options, TContext context)
succeeded &= ValidateFile(context, options.OutputFilePath, shouldExist: null);
succeeded &= ValidateFile(context, options.ConfigurationFilePath, shouldExist: true);
succeeded &= ValidateFiles(context, options.PluginFilePaths, shouldExist: true);
succeeded &= ValidateFile(context, options.BaselineSarifFile, shouldExist: true);
succeeded &= ValidateInvocationPropertiesToLog(context, options.InvocationPropertiesToLog);
succeeded &= ValidateOutputFileCanBeCreated(context, options.OutputFilePath, options.Force);
succeeded &= options.ValidateOutputOptions(context);

if (!succeeded)
{
Expand Down
68 changes: 68 additions & 0 deletions src/Sarif.Driver/Sdk/PluginDriverCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,14 @@

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;

using Microsoft.CodeAnalysis.Sarif.Baseline.ResultMatching;

using Newtonsoft.Json;

namespace Microsoft.CodeAnalysis.Sarif.Driver
{
public abstract class PluginDriverCommand<T> : DriverCommand<T>
Expand All @@ -30,5 +36,67 @@ public IEnumerable<Assembly> RetrievePluginAssemblies(IEnumerable<Assembly> defa

return assemblies;
}

protected virtual void ProcessBaseline(IAnalysisContext context, T driverOptions, IFileSystem fileSystem)
{
if (!(driverOptions is AnalyzeOptionsBase options))
{
return;
}

if (string.IsNullOrEmpty(options.BaselineSarifFile) || string.IsNullOrEmpty(options.OutputFilePath))
{
return;
}

var serializer = new JsonSerializer
{
Formatting = options.PrettyPrint || (!options.PrettyPrint && !options.Minify) ?
Formatting.Indented :
Formatting.None
};

SarifLog baselineFile;
using (var reader = new JsonTextReader(new StreamReader(fileSystem.FileOpenRead(options.BaselineSarifFile))))
{
baselineFile = serializer.Deserialize<SarifLog>(reader);
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you can replace this by:

baselineFile = SarifLog.Load(optons.BaselineSarifFile)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can I inject a mock IFileSystem to SarifLog for test cases?


SarifLog currentSarifLog;
using (var reader = new JsonTextReader(new StreamReader(fileSystem.FileOpenRead(options.OutputFilePath))))
{
currentSarifLog = serializer.Deserialize<SarifLog>(reader);
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

use this:

currentSarifLog = SarifLog.Load(xyz)


SarifLog baseline;
try
{
ISarifLogMatcher matcher = ResultMatchingBaselinerFactory.GetDefaultResultMatchingBaseliner();
baseline = matcher.Match(new SarifLog[] { baselineFile }, new SarifLog[] { currentSarifLog }).First();
}
catch (Exception ex)
{
throw new ExitApplicationException<ExitReason>(DriverResources.MSG_UnexpectedApplicationExit, ex)
{
ExitReason = ExitReason.ExceptionProcessingBaseline
};
}

try
{
string targetFile = options.Inline ? options.BaselineSarifFile : options.OutputFilePath;
using (var writer = new JsonTextWriter(new StreamWriter(fileSystem.FileCreate(targetFile))))
{
serializer.Serialize(writer, baseline);
}
}
catch (Exception ex)
{
throw new ExitApplicationException<ExitReason>(DriverResources.MSG_UnexpectedApplicationExit, ex)
{
ExitReason = ExitReason.ExceptionWritingToLogFile
};
}
}
}
}
1 change: 1 addition & 0 deletions src/Sarif/RuntimeConditions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ public enum RuntimeConditions : uint
ExceptionAccessingFile = 0x4000,
ExceptionInstantiatingSkimmers = 0x8000,
OutputFileAlreadyExists = 0x10000,
ExceptionProcessingBaseline = 0x20000,

// Non-fatal conditions
UnassignedNonfatal = 0x01F00000,
Expand Down
9 changes: 5 additions & 4 deletions src/Test.UnitTests.Sarif.Driver/DriverExtensionMethodsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,26 +243,27 @@ public void ValidatingMultipleFilesOutputOptions_ProducesExpectedResults()
public void ValidateAnalyzeOutputOptions_ProducesExpectedResults()
{
AnalyzeOptionsBase analyzeOptionsBase = new TestAnalyzeOptions();
TestAnalysisContext context = new TestAnalysisContext();

// quiet false, output empty
analyzeOptionsBase.Quiet = false;
analyzeOptionsBase.OutputFilePath = null;
Assert.True(analyzeOptionsBase.ValidateOutputOptions());
Assert.True(analyzeOptionsBase.ValidateOutputOptions(context));

// quiet false, output non-empty
analyzeOptionsBase.Quiet = false;
analyzeOptionsBase.OutputFilePath = "doodle";
Assert.True(analyzeOptionsBase.ValidateOutputOptions());
Assert.True(analyzeOptionsBase.ValidateOutputOptions(context));

// quiet true, output empty
analyzeOptionsBase.Quiet = true;
analyzeOptionsBase.OutputFilePath = null;
Assert.False(analyzeOptionsBase.ValidateOutputOptions());
Assert.False(analyzeOptionsBase.ValidateOutputOptions(context));

// quiet true, output non-empty
analyzeOptionsBase.Quiet = true;
analyzeOptionsBase.OutputFilePath = "doodle";
Assert.True(analyzeOptionsBase.ValidateOutputOptions());
Assert.True(analyzeOptionsBase.ValidateOutputOptions(context));
}

private class ValidateOutputFormatOptionsTestCase
Expand Down
57 changes: 57 additions & 0 deletions src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,22 @@ public void ExceptionRaisedInvokingAnalyze()
);
}

[Fact]
public void ExceptionRaisedProcessingBaseline()
{
var options = new TestAnalyzeOptions()
{
TestRuleBehaviors = TestRuleBehaviors.RaiseExceptionProcessingBaseline,
TargetFileSpecifiers = new string[] { GetThisTestAssemblyFilePath() },
};

ExceptionTestHelper(
RuntimeConditions.ExceptionProcessingBaseline,
expectedExitReason: ExitReason.ExceptionProcessingBaseline,
analyzeOptions: options
);
}


[Fact]
public void ExceptionRaisedInvokingAnalyze_PersistInnerException()
Expand Down Expand Up @@ -452,6 +468,47 @@ public void MissingOutputFile()
}
}

[Fact]
public void MissingBaselineFile()
{
string outputFilePath = Path.GetTempFileName() + ".sarif";
string baselineFilePath = Path.GetTempFileName() + ".sarif";

var options = new TestAnalyzeOptions()
{
TargetFileSpecifiers = new string[] { GetThisTestAssemblyFilePath() },
OutputFilePath = outputFilePath,
BaselineSarifFile = baselineFilePath
};

ExceptionTestHelper(
RuntimeConditions.MissingFile,
expectedExitReason: ExitReason.InvalidCommandLineOption,
analyzeOptions: options);
}

[Fact]
public void BaselineWithoutOutputFile()
{
string path = Path.GetTempFileName() + ".sarif";

using (FileStream stream = File.Create(path, 1, FileOptions.DeleteOnClose))
{
var options = new TestAnalyzeOptions()
{
TargetFileSpecifiers = new string[] { GetThisTestAssemblyFilePath() },
Quiet = true,
OutputFilePath = null,
BaselineSarifFile = path
};

ExceptionTestHelper(
RuntimeConditions.InvalidCommandLineOption,
expectedExitReason: ExitReason.InvalidCommandLineOption,
analyzeOptions: options);
}
}

[Fact]
public void AnalyzeCommandBase_ReportsErrorOnInvalidInvocationPropertyName()
{
Expand Down
11 changes: 11 additions & 0 deletions src/Test.UnitTests.Sarif.Driver/TestAnalyzeCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,5 +59,16 @@ protected override void ValidateOptions(TestAnalysisContext context, TestAnalyze

base.ValidateOptions(context, options);
}

protected override void ProcessBaseline(IAnalysisContext context, TestAnalyzeOptions options, IFileSystem fileSystem)
{
if (context.Policy.GetProperty(TestRule.Behaviors).HasFlag(TestRuleBehaviors.RaiseExceptionProcessingBaseline))
{
context.RuntimeErrors |= RuntimeConditions.ExceptionProcessingBaseline;
ThrowExitApplicationException((TestAnalysisContext)context, ExitReason.ExceptionProcessingBaseline);
}

base.ProcessBaseline(context, options, fileSystem);
}
}
}
11 changes: 11 additions & 0 deletions src/Test.UnitTests.Sarif.Driver/TestMultithreadedAnalyzeCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,5 +70,16 @@ protected override TestAnalysisContext DetermineApplicabilityAndAnalyze(TestAnal

return base.DetermineApplicabilityAndAnalyze(context, skimmers, disabledSkimmers);
}

protected override void ProcessBaseline(IAnalysisContext context, TestAnalyzeOptions options, IFileSystem fileSystem)
{
if (context.Policy.GetProperty(TestRule.Behaviors).HasFlag(TestRuleBehaviors.RaiseExceptionProcessingBaseline))
{
context.RuntimeErrors |= RuntimeConditions.ExceptionProcessingBaseline;
ThrowExitApplicationException((TestAnalysisContext)context, ExitReason.ExceptionProcessingBaseline);
}

base.ProcessBaseline(context, options, fileSystem);
}
}
}
4 changes: 3 additions & 1 deletion src/Test.Utilities.Sarif/TestRuleBehaviors.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ public enum TestRuleBehaviors
RegardAnalysisTargetAsInvalid = 0x2000,

// Assume one or more options are invalid
RegardOptionsAsInvalid = 0x4000
RegardOptionsAsInvalid = 0x4000,

RaiseExceptionProcessingBaseline = 0x8000
}
}