Skip to content

Commit

Permalink
Cache Results of IsCommented (#369)
Browse files Browse the repository at this point in the history 
* Improve Scope Match performance

* Fix

* Update TextContainer.cs

* Test fix

* Update TestAnalyzeCmd.cs

* Update TextContainer.cs

* Update TextContainer.cs

* Update TextContainer.cs

* Update TextContainer.cs

* Clean up text container

* Update RuleProcessor.cs

* Use Globs for file exclusions

* Update AnalyzeCommand.cs

* Update CLICmdOptions.cs

* Add none to disable

* Update GetTagsCommand.cs

* Fix build

* Fix Text Contains

Respect parallel in rule processor
Update descriptions for command options
Reduce sleep frequency

* Fix filter tests

* Update TestGetTagsCmd.cs

* Repro of null rules in match

* Remove TagTest

GetTags seems to perform the same task.

* Remove TagTest command

* Limit parallelization to decrease timeouts

* Fix test

* Fix regexword implementation

* Clean up rules

Improve some rules, remove some unneeded fields.

* Fix Rules and RulePacker

Fix Regex Word behavior

* Fix Pack Rules

* Narrow media regexes

* Fix tests

* Fix test

* Improve TagDiff performane

* Fix async analyze

* Update TestAnalyzeCmd.cs

* Update TestAnalyzeCmd.cs

* Update TestAnalyzeCmd.cs

* Workaround for IndexOf on Windows

* Update OatSubstringIndexOperation.cs

* Remove multithread enumerating

* Update TextContainer.cs

* Update RuleProcessor.cs

* Don't precheck matches count.

* Respect numcontextlines

* Update AnalyzeBenchmark.cs

* Add Multi path option

* Update TestGetTagsCmd.cs

* Fix verifier

* simplify regex word construction

* Better simplify

* Update TestGetTagsCmd.cs

* Update test numbers to match fixed behavior of regex-word

* Update Ruleset.cs

* Fix test numbers with fixed regex word

* Update TagDiffCommand.cs

* Make Get-Tags an option of Analyze

GetTags and Analyze were mostly duplicative so instead the GetTags behavior is now provided by giving `-t`  or `TagsOnly` to Analyze.

* bump to RC

* fixes

* Rename tests to accurately reflect using analyze command

* Support multiple input for TagDiff via comma separated

* Remove test for removed functionality

* Add missing comments

* More comments.

* Make FilePathExclusions parsed automatically.

* Roslynator Changes

* Respond to comments.

* Fix tests

* Fix tests

* More Roslynator Changes
  • Loading branch information
@gfs
gfs committed May 24, 2021
1 parent 69410ed commit ca2efda
Show file tree
Hide file tree
Showing 125 changed files with 2,443 additions and 5,739 deletions.
2 changes: 1 addition & 1 deletion AppInspector.CLI/AppInspector.CLI.csproj
View file
Expand Up @@ -66,7 +66,7 @@

<ItemGroup>
<PackageReference Include="DotLiquid" Version="2.1.436" />
<PackageReference Include="NLog" Version="4.7.9" />
<PackageReference Include="NLog" Version="4.7.10" />
<PackageReference Include="ShellProgressBar" Version="5.1.0" />
<PackageReference Update="Nerdbank.GitVersioning" Version="3.4.194" />
</ItemGroup>
Expand Down
84 changes: 21 additions & 63 deletions AppInspector.CLI/CLICmdOptions.cs
View file
Expand Up @@ -3,6 +3,7 @@

using CommandLine;
using Microsoft.ApplicationInspector.Commands;
using System.Collections.Generic;

namespace Microsoft.ApplicationInspector.CLI
{
Expand All @@ -19,51 +20,14 @@ public class CLICommandOptions : CommandOptions
public string OutputFileFormat { get; set; } = "text";
}

/// <summary>
/// CLI command distinct arguments
/// </summary>
[Verb("get-tags", HelpText = "Inspect source directory/file/compressed file (.tgz|zip) against defined characteristics for unique tags")]
public class CLIGetTagsCommandOptions : CLICommandOptions
{
[Option('s', "source-path", Required = true, HelpText = "Source file or directory to inspect")]
public string? SourcePath { get; set; }

[Option('r', "custom-rules-path", Required = false, HelpText = "Custom rules file or directory path")]
public string? CustomRulesPath { get; set; }

[Option('i', "ignore-default-rules", Required = false, HelpText = "Exclude default rules bundled with application", Default = false)]
public bool IgnoreDefaultRules { get; set; }

[Option('c', "confidence-filters", Required = false, HelpText = "Output only matches with specified confidence <value>,<value> [high|medium|low]", Default = "high,medium")]
public string ConfidenceFilters { get; set; } = "high,medium";

[Option('k', "file-path-exclusions", Required = false, HelpText = "Exclude source files (none|default: sample,example,test,docs,lib,.vs,.git)", Default = "sample,example,test,docs,lib,.vs,.git")]
public string FilePathExclusions { get; set; } = "sample,example,test,docs,.vs,.git";

[Option("file-timeout", Required = false, HelpText = "If set, maximum amount of time in milliseconds to allow for processing each file.", Default = 0)]
public int FileTimeOut { get; set; } = 0;

[Option("processing-timeout", Required = false, HelpText = "If set, maximum amount of time in milliseconds to allow for processing overall.", Default = 0)]
public int ProcessingTimeOut { get; set; } = 0;

[Option("single-threaded", Required = false, HelpText = "Disables parallel processing.")]
public bool SingleThread { get; set; }

[Option("no-show-progress", Required = false, HelpText = "Disable progress information.")]
public bool NoShowProgressBar { get; set; }

[Option("scan-unknown-filetypes", Required = false, HelpText = "Scan files of unknown types.")]
public bool ScanUnknownTypes { get; set; }
}

/// <summary>
/// CLI command distinct arguments
/// </summary>
[Verb("analyze", HelpText = "Inspect source directory/file/compressed file (.tgz|zip) against defined characteristics")]
public class CLIAnalyzeCmdOptions : CLICommandOptions
{
[Option('s', "source-path", Required = true, HelpText = "Source file or directory to inspect")]
public string? SourcePath { get; set; }
[Option('s', "source-path", Required = true, HelpText = "Source file or directory to inspect, comma separated", Separator = ',')]
public IEnumerable<string> SourcePath { get; set; } = new string[0];

[Option('r', "custom-rules-path", Required = false, HelpText = "Custom rules file or directory path")]
public string? CustomRulesPath { get; set; }
Expand All @@ -74,17 +38,18 @@ public class CLIAnalyzeCmdOptions : CLICommandOptions
[Option('c', "confidence-filters", Required = false, HelpText = "Output only matches with specified confidence <value>,<value> [high|medium|low]", Default = "high,medium")]
public string ConfidenceFilters { get; set; } = "high,medium";

[Option('k', "file-path-exclusions", Required = false, HelpText = "Exclude source files (none|default: sample,example,test,docs,lib,.vs,.git)", Default = "sample,example,test,docs,lib,.vs,.git")]
public string FilePathExclusions { get; set; } = "sample,example,test,docs,.vs,.git";
[Option('k', "file-path-exclusions", Required = false, HelpText = "Exclude source files that match glob patterns. Example: \".git,**Tests**\". Use \"none\" to disable.", Default = "*/bin,*/obj,*/.vs,*/.git", Separator = ',')]

public IEnumerable<string> FilePathExclusions { get; set; } = new string[] { };

[Option('f', "output-file-format", Required = false, HelpText = "Output format [html|json|text]", Default = "html")]
public new string OutputFileFormat { get; set; } = "html";

[Option('e', "text-format", Required = false, HelpText = "Match text format specifiers", Default = "Tag:%T,Rule:%N,Ruleid:%R,Confidence:%X,File:%F,Sourcetype:%t,Line:%L,Sample:%m")]
public string TextOutputFormat { get; set; } = "Tag:%T,Rule:%N,Ruleid:%R,Confidence:%X,File:%F,Sourcetype:%t,Line:%L,Sample:%m";

[Option("file-timeout", Required = false, HelpText = "If set, maximum amount of time in milliseconds to allow for processing each file.", Default = 0)]
public int FileTimeOut { get; set; } = 0;
[Option("file-timeout", Required = false, HelpText = "If set, maximum amount of time in milliseconds to allow for processing each file.", Default = 60000)]
public int FileTimeOut { get; set; } = 60000;

[Option("processing-timeout", Required = false, HelpText = "If set, maximum amount of time in milliseconds to allow for processing overall.", Default = 0)]
public int ProcessingTimeOut { get; set; } = 0;
Expand All @@ -100,44 +65,37 @@ public class CLIAnalyzeCmdOptions : CLICommandOptions

[Option("scan-unknown-filetypes", Required = false, HelpText = "Scan files of unknown types.")]
public bool ScanUnknownTypes { get; set; }

[Option('t',"tags-only", Required = false, HelpText = "Only get tags (no detailed match data).")]
public bool TagsOnly { get; set; }
}

[Verb("tagdiff", HelpText = "Compares unique tag values between two source paths")]
public class CLITagDiffCmdOptions : CLICommandOptions
{
[Option("src1", Required = true, HelpText = "Source 1 to compare")]
public string? SourcePath1 { get; set; }
[Option("src1", Required = true, HelpText = "Source 1 to compare (commaa separated)")]
public IEnumerable<string> SourcePath1 { get; set; } = System.Array.Empty<string>();

[Option("src2", Required = true, HelpText = "Source 2 to compare")]
public string? SourcePath2 { get; set; }
[Option("src2", Required = true, HelpText = "Source 2 to compare (commaa separated)")]
public IEnumerable<string> SourcePath2 { get; set; } = System.Array.Empty<string>();

[Option('t', "test-type", Required = false, HelpText = "Type of test to run [equality|inequality]", Default = "equality")]
public string TestType { get; set; } = "equality";

[Option('k', "file-path-exclusions", Required = false, HelpText = "Exclude source files (none|default: sample,example,test,docs,.vs,.git)", Default = "sample,example,test,docs,.vs,.git")]
public string FilePathExclusions { get; set; } = "sample,example,test,docs,.vs,.git";
[Option('k', "file-path-exclusions", Required = false, HelpText = "Exclude source files that match glob patterns. Example: \".git,**Tests**\". Use \"none\" to disable.", Default = "*/bin,*/obj,*/.vs,*/.git", Separator = ',')]
public IEnumerable<string> FilePathExclusions { get; set; } = new string[] { };

[Option('r', "custom-rules-path", Required = false, HelpText = "Custom rules file or directory path")]
public string? CustomRulesPath { get; set; }

[Option('i', "ignore-default-rules", Required = false, HelpText = "Exclude default rules bundled with application", Default = false)]
public bool IgnoreDefaultRules { get; set; }
}

[Verb("tagtest", HelpText = "Test (T/F) for presence of custom rule set in source")]
public class CLITagTestCmdOptions : CLICommandOptions
{
[Option('s', "source-path", Required = true, HelpText = "Source file or directory to inspect")]
public string? SourcePath { get; set; }
[Option("file-timeout", Required = false, HelpText = "If set, maximum amount of time in milliseconds to allow for processing each file.", Default = 60000)]
public int FileTimeOut { get; set; } = 60000;

[Option('t', "test-type", Required = false, HelpText = "Test to perform [rulespresent|rulesnotpresent] ", Default = "rulespresent")]
public string TestType { get; set; } = "rulespresent";

[Option('r', "custom-rules-path", Required = false, HelpText = "Custom rules file or directory path")]
public string? CustomRulesPath { get; set; }

[Option('k', "file-path-exclusions", Required = false, HelpText = "Exclude source files (none|default: sample,example,test,docs,.vs,.git)", Default = "sample,example,test,docs,.vs,.git")]
public string FilePathExclusions { get; set; } = "sample,example,test,docs,.vs,.git";
[Option("processing-timeout", Required = false, HelpText = "If set, maximum amount of time in milliseconds to allow for processing overall.", Default = 0)]
public int ProcessingTimeOut { get; set; } = 0;
}

[Verb("exporttags", HelpText = "Export unique rule tags to view what code features may be detected")]
Expand Down
134 changes: 14 additions & 120 deletions AppInspector.CLI/Program.cs
View file
Expand Up @@ -6,14 +6,15 @@
using NLog;
using ShellProgressBar;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace Microsoft.ApplicationInspector.CLI
{
public class Program
public static class Program
{
/// <summary>
/// CLI program entry point which defines command verbs and options to running
Expand All @@ -30,19 +31,15 @@ public static int Main(string[] args)
{
var argsResult = Parser.Default.ParseArguments<CLIAnalyzeCmdOptions,
CLITagDiffCmdOptions,
CLITagTestCmdOptions,
CLIExportTagsCmdOptions,
CLIVerifyRulesCmdOptions,
CLIPackRulesCmdOptions,
CLIGetTagsCommandOptions>(args)
CLIPackRulesCmdOptions>(args)
.MapResult(
(CLIAnalyzeCmdOptions cliOptions) => VerifyOutputArgsRun(cliOptions),
(CLITagDiffCmdOptions cliOptions) => VerifyOutputArgsRun(cliOptions),
(CLITagTestCmdOptions cliOptions) => VerifyOutputArgsRun(cliOptions),
(CLIExportTagsCmdOptions cliOptions) => VerifyOutputArgsRun(cliOptions),
(CLIVerifyRulesCmdOptions cliOptions) => VerifyOutputArgsRun(cliOptions),
(CLIPackRulesCmdOptions cliOptions) => VerifyOutputArgsRun(cliOptions),
(CLIGetTagsCommandOptions cliOptions) => VerifyOutputArgsRun(cliOptions),
errs => 2
);

Expand Down Expand Up @@ -99,17 +96,6 @@ private static int VerifyOutputArgsRun(CLITagDiffCmdOptions options)
CommonOutputChecks(options);
return RunTagDiffCommand(options);
}

private static int VerifyOutputArgsRun(CLITagTestCmdOptions options)
{
Logger logger = Utils.SetupLogging(options, true);
WriteOnce.Log = logger;
options.Log = logger;

CommonOutputChecks(options);
return RunTagTestCommand(options);
}

private static int VerifyOutputArgsRun(CLIExportTagsCmdOptions options)
{
Logger logger = Utils.SetupLogging(options, true);
Expand Down Expand Up @@ -155,16 +141,6 @@ private static int VerifyOutputArgsRun(CLIPackRulesCmdOptions options)
return RunPackRulesCommand(options);
}

private static int VerifyOutputArgsRun(CLIGetTagsCommandOptions options)
{
Logger logger = Utils.SetupLogging(options, true);
WriteOnce.Log = logger;
options.Log = logger;

CommonOutputChecks(options);
return RunGetTagsCommand(options);
}

private static int VerifyOutputArgsRun(CLIAnalyzeCmdOptions options)
{
Logger logger = Utils.SetupLogging(options, true);
Expand Down Expand Up @@ -229,12 +205,12 @@ private static void CommonOutputChecks(CLICommandOptions options)
//validate output is not empty if no file output specified
if (string.IsNullOrEmpty(options.OutputFilePath))
{
if (options.ConsoleVerbosityLevel.ToLower() == "none")
if (string.Equals(options.ConsoleVerbosityLevel, "none", StringComparison.OrdinalIgnoreCase))
{
WriteOnce.Error(MsgHelp.GetString(MsgHelp.ID.CMD_NO_OUTPUT));
throw new Exception(MsgHelp.GetString(MsgHelp.ID.CMD_NO_OUTPUT));
}
else if (options.ConsoleVerbosityLevel.ToLower() == "low")
else if (string.Equals(options.ConsoleVerbosityLevel, "low", StringComparison.OrdinalIgnoreCase))
{
WriteOnce.SafeLog("Verbosity set low. Detailed output limited.", NLog.LogLevel.Info);
}
Expand Down Expand Up @@ -266,77 +242,11 @@ private static void ValidFileWritePath(string filePath)

#region RunCmdsWriteResults

private static int RunGetTagsCommand(CLIGetTagsCommandOptions cliOptions)
{
GetTagsCommand command = new GetTagsCommand(new GetTagsCommandOptions()
{
SourcePath = cliOptions.SourcePath ?? "",
CustomRulesPath = cliOptions.CustomRulesPath ?? "",
IgnoreDefaultRules = cliOptions.IgnoreDefaultRules,
ConfidenceFilters = cliOptions.ConfidenceFilters,
FilePathExclusions = cliOptions.FilePathExclusions,
ConsoleVerbosityLevel = cliOptions.ConsoleVerbosityLevel,
Log = cliOptions.Log,
SingleThread = cliOptions.SingleThread,
NoShowProgress = cliOptions.NoShowProgressBar,
FileTimeOut = cliOptions.FileTimeOut,
ProcessingTimeOut = cliOptions.ProcessingTimeOut,
ScanUnknownTypes = cliOptions.ScanUnknownTypes
});


if (!cliOptions.NoShowProgressBar)
{
WriteOnce.PauseConsoleOutput = true;
}

GetTagsResult getTagsResult = command.GetResult();

if (cliOptions.NoShowProgressBar)
{
ResultsWriter.Write(getTagsResult, cliOptions);
}
else
{
var done = false;

_ = Task.Factory.StartNew(() =>
{
ResultsWriter.Write(getTagsResult, cliOptions);
done = true;
});

var options = new ProgressBarOptions
{
ForegroundColor = ConsoleColor.Yellow,
ForegroundColorDone = ConsoleColor.DarkGreen,
BackgroundColor = ConsoleColor.DarkGray,
BackgroundCharacter = '\u2593',
DisableBottomPercentage = true
};

using (var pbar = new IndeterminateProgressBar("Writing Result Files.", options))
{
while (!done)
{
Thread.Sleep(10);
}
pbar.Message = $"Results written.";

pbar.Finished();
}
}

WriteOnce.PauseConsoleOutput = false;

return (int)getTagsResult.ResultCode;
}

private static int RunAnalyzeCommand(CLIAnalyzeCmdOptions cliOptions)
{
AnalyzeCommand command = new AnalyzeCommand(new AnalyzeOptions()
{
SourcePath = cliOptions.SourcePath ?? "",
SourcePath = cliOptions.SourcePath ?? Array.Empty<string>(),
CustomRulesPath = cliOptions.CustomRulesPath ?? "",
IgnoreDefaultRules = cliOptions.IgnoreDefaultRules,
ConfidenceFilters = cliOptions.ConfidenceFilters,
Expand All @@ -348,7 +258,8 @@ private static int RunAnalyzeCommand(CLIAnalyzeCmdOptions cliOptions)
FileTimeOut = cliOptions.FileTimeOut,
ProcessingTimeOut = cliOptions.ProcessingTimeOut,
ContextLines = cliOptions.ContextLines,
ScanUnknownTypes = cliOptions.ScanUnknownTypes
ScanUnknownTypes = cliOptions.ScanUnknownTypes,
TagsOnly = cliOptions.TagsOnly
});

if (!cliOptions.NoShowProgressBar)
Expand Down Expand Up @@ -385,12 +296,13 @@ private static int RunAnalyzeCommand(CLIAnalyzeCmdOptions cliOptions)
{
while (!done)
{
Thread.Sleep(10);
Thread.Sleep(100);
}
pbar.Message = $"Results written.";
pbar.Message = "Results written.";

pbar.Finished();
}
Console.Write(Environment.NewLine);
}

WriteOnce.PauseConsoleOutput = false;
Expand All @@ -402,8 +314,8 @@ private static int RunTagDiffCommand(CLITagDiffCmdOptions cliOptions)
{
TagDiffCommand command = new TagDiffCommand(new TagDiffOptions()
{
SourcePath1 = cliOptions.SourcePath1 ?? "",
SourcePath2 = cliOptions.SourcePath2 ?? "",
SourcePath1 = cliOptions.SourcePath1,
SourcePath2 = cliOptions.SourcePath2,
CustomRulesPath = cliOptions.CustomRulesPath,
IgnoreDefaultRules = cliOptions.IgnoreDefaultRules,
FilePathExclusions = cliOptions.FilePathExclusions,
Expand All @@ -417,25 +329,7 @@ private static int RunTagDiffCommand(CLITagDiffCmdOptions cliOptions)

return (int)tagDiffResult.ResultCode;
}

private static int RunTagTestCommand(CLITagTestCmdOptions cliOptions)
{
TagTestCommand command = new TagTestCommand(new TagTestOptions()
{
SourcePath = cliOptions.SourcePath,
CustomRulesPath = cliOptions.CustomRulesPath,
FilePathExclusions = cliOptions.FilePathExclusions,
TestType = cliOptions.TestType,
ConsoleVerbosityLevel = cliOptions.ConsoleVerbosityLevel,
Log = cliOptions.Log
});

TagTestResult tagTestCommand = command.GetResult();
ResultsWriter.Write(tagTestCommand, cliOptions);

return (int)tagTestCommand.ResultCode;
}


private static int RunExportTagsCommand(CLIExportTagsCmdOptions cliOptions)
{
ExportTagsCommand command = new ExportTagsCommand(new ExportTagsOptions()
Expand Down

0 comments on commit ca2efda

Please sign in to comment.