Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add async #354

Merged
merged 6 commits into from
May 6, 2021
Merged

Add async #354

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
89cfe6e
Add Async Versions of GetResult appropriate for WASM use
gfs May 6, 2021
0b0c8e4
Fix GetTags command uniqueness
gfs May 6, 2021
2124833
Update RuleProcessor.cs
gfs May 6, 2021
de8db68
Build fix
gfs May 6, 2021
a07d79b
Fix #353
gfs May 6, 2021
8e5efeb
Add async tests
gfs May 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
131 changes: 129 additions & 2 deletions AppInspector/Commands/AnalyzeCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,6 @@ private void ConfigRules()
var rpo = new RuleProcessorOptions()
{
logger = _options.Log,
uniqueMatches = false,
treatEverythingAsCode = _options.TreatEverythingAsCode,
confidenceFilter = _confidence
};
Expand Down Expand Up @@ -389,6 +388,81 @@ void ProcessAndAddToMetadata(FileEntry file)
}
}

public async Task<AnalyzeResult.ExitCode> PopulateRecordsAsync(CancellationToken cancellationToken, AnalyzeOptions opts)
{
WriteOnce.SafeLog("AnalyzeCommand::PopulateRecordsAsync", LogLevel.Trace);

if (_rulesProcessor is null)
{
return AnalyzeResult.ExitCode.CriticalError;
}
await foreach (var entry in GetFileEntriesAsync(opts))
{
if (cancellationToken.IsCancellationRequested) { return AnalyzeResult.ExitCode.Canceled; }
await ProcessAndAddToMetadata(entry, cancellationToken);
}

return AnalyzeResult.ExitCode.Success;

async Task ProcessAndAddToMetadata(FileEntry file, CancellationToken cancellationToken)
{
var fileRecord = new FileRecord() { FileName = file.FullPath, ModifyTime = file.ModifyTime, CreateTime = file.CreateTime, AccessTime = file.AccessTime };

var sw = new Stopwatch();
sw.Start();

if (_fileExclusionList.Any(x => file.FullPath.ToLower().Contains(x)))
{
WriteOnce.SafeLog(MsgHelp.FormatString(MsgHelp.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, fileRecord.FileName), LogLevel.Debug);
fileRecord.Status = ScanState.Skipped;
}
else
{
if (IsBinary(file.Content))
{
WriteOnce.SafeLog(MsgHelp.FormatString(MsgHelp.ID.ANALYZE_EXCLUDED_BINARY, fileRecord.FileName), LogLevel.Debug);
fileRecord.Status = ScanState.Skipped;
}
else
{
_ = _metaDataHelper?.FileExtensions.TryAdd(Path.GetExtension(file.FullPath).Replace('.', ' ').TrimStart(), 0);

LanguageInfo languageInfo = new LanguageInfo();

if (Language.FromFileName(file.FullPath, ref languageInfo))
{
_metaDataHelper?.AddLanguage(languageInfo.Name);
}
else
{
_metaDataHelper?.AddLanguage("Unknown");
languageInfo = new LanguageInfo() { Extensions = new string[] { Path.GetExtension(file.FullPath) }, Name = "Unknown" };
}


var results = await _rulesProcessor.AnalyzeFileAsync(file, languageInfo, cancellationToken);
fileRecord.Status = ScanState.Analyzed;

if (results.Any())
{
fileRecord.Status = ScanState.Affected;
fileRecord.NumFindings = results.Count;
}
foreach (var matchRecord in results)
{
_metaDataHelper?.AddMatchRecord(matchRecord);
}
}
}

sw.Stop();

fileRecord.ScanTime = sw.Elapsed;

_metaDataHelper?.Files.Add(fileRecord);
}
}

public List<FileEntry> GetFileEntries(AnalyzeOptions opts)
{
WriteOnce.SafeLog("AnalyzeCommand::GetFileEntries", LogLevel.Trace);
Expand All @@ -411,6 +485,21 @@ public List<FileEntry> GetFileEntries(AnalyzeOptions opts)
}


public async IAsyncEnumerable<FileEntry> GetFileEntriesAsync(AnalyzeOptions opts)
{
WriteOnce.SafeLog("AnalyzeCommand::GetFileEntriesAsync", LogLevel.Trace);

Extractor extractor = new();
foreach (var srcFile in _srcfileList ?? Array.Empty<string>())
{
await foreach (var entry in extractor.ExtractAsync(srcFile, new ExtractorOptions() { Parallel = false }))
{
yield return entry;
}
}
}


// Follows Perl's model, if there are NULs or too many non printable characters, this is probably a binary file
private bool IsBinary(Stream fileContents)
{
Expand Down Expand Up @@ -444,6 +533,44 @@ private bool IsBinary(Stream fileContents)
return false;
}

public async Task<AnalyzeResult> GetResultAsync(CancellationToken cancellationToken)
{
WriteOnce.SafeLog("AnalyzeCommand::GetResultAsync", LogLevel.Trace);
WriteOnce.Operation(MsgHelp.FormatString(MsgHelp.ID.CMD_RUNNING, "Analyze"));
AnalyzeResult analyzeResult = new AnalyzeResult()
{
AppVersion = Utils.GetVersionString()
};

var exitCode = await PopulateRecordsAsync(cancellationToken, _options);

//wrapup result status
if (_metaDataHelper?.Files.All(x => x.Status == ScanState.Skipped) ?? false)
{
WriteOnce.Error(MsgHelp.GetString(MsgHelp.ID.ANALYZE_NOSUPPORTED_FILETYPES));
analyzeResult.ResultCode = AnalyzeResult.ExitCode.NoMatches;
}
else if (_metaDataHelper?.Matches.Count == 0)
{
WriteOnce.Error(MsgHelp.GetString(MsgHelp.ID.ANALYZE_NOPATTERNS));
analyzeResult.ResultCode = AnalyzeResult.ExitCode.NoMatches;
}
else if (_metaDataHelper != null && _metaDataHelper.Metadata != null)
{
_metaDataHelper.Metadata.DateScanned = DateScanned.ToString();
_metaDataHelper.PrepareReport();
analyzeResult.Metadata = _metaDataHelper.Metadata; //replace instance with metadatahelper processed one
analyzeResult.ResultCode = AnalyzeResult.ExitCode.Success;
}

if (cancellationToken.IsCancellationRequested)
{
analyzeResult.ResultCode = AnalyzeResult.ExitCode.Canceled;
}

return analyzeResult;
}

/// <summary>
/// Main entry point to start analysis from CLI; handles setting up rules, directory enumeration
/// file type detection and handoff
Expand All @@ -452,7 +579,7 @@ private bool IsBinary(Stream fileContents)
/// <returns></returns>
public AnalyzeResult GetResult()
{
WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);
WriteOnce.SafeLog("AnalyzeCommand::GetResult", LogLevel.Trace);
WriteOnce.Operation(MsgHelp.FormatString(MsgHelp.ID.CMD_RUNNING, "Analyze"));
AnalyzeResult analyzeResult = new AnalyzeResult()
{
Expand Down
137 changes: 133 additions & 4 deletions AppInspector/Commands/GetTagsCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,6 @@ private void ConfigRules()
var rpo = new RuleProcessorOptions()
{
logger = _options.Log,
uniqueMatches = true,
treatEverythingAsCode = _options.TreatEverythingAsCode,
confidenceFilter = _confidence
};
Expand Down Expand Up @@ -369,7 +368,7 @@ void ProcessAndAddToMetadata(FileEntry file)
if (opts.FileTimeOut > 0)
{
using var cts = new CancellationTokenSource();
var t = Task.Run(() => results = _rulesProcessor.AnalyzeFile(file, languageInfo, null), cts.Token);
var t = Task.Run(() => results = _rulesProcessor.AnalyzeFile(file, languageInfo, _metaDataHelper?.UniqueTags.Keys), cts.Token);
if (!t.Wait(new TimeSpan(0, 0, 0, 0, opts.FileTimeOut)))
{
WriteOnce.Error($"{file.FullPath} timed out.");
Expand All @@ -383,7 +382,7 @@ void ProcessAndAddToMetadata(FileEntry file)
}
else
{
results = _rulesProcessor.AnalyzeFile(file, languageInfo, null);
results = _rulesProcessor.AnalyzeFile(file, languageInfo, _metaDataHelper?.UniqueTags.Keys);
fileRecord.Status = ScanState.Analyzed;
}

Expand Down Expand Up @@ -437,6 +436,136 @@ private bool IsBinary(Stream fileContents)
return false;
}

public async IAsyncEnumerable<FileEntry> GetFileEntriesAsync(GetTagsCommandOptions opts)
{
WriteOnce.SafeLog("AnalyzeCommand::GetFileEntriesAsync", LogLevel.Trace);

Extractor extractor = new();
foreach (var srcFile in _srcfileList ?? Array.Empty<string>())
{
await foreach (var entry in extractor.ExtractAsync(srcFile, new ExtractorOptions() { Parallel = false }))
{
yield return entry;
}
}
}

public async Task<GetTagsResult.ExitCode> PopulateRecordsAsync(CancellationToken cancellationToken, GetTagsCommandOptions opts)
{
WriteOnce.SafeLog("GetTagsCommand::PopulateRecordsAsync", LogLevel.Trace);

if (_rulesProcessor is null)
{
return GetTagsResult.ExitCode.CriticalError;
}
await foreach (var entry in GetFileEntriesAsync(opts))
{
if (cancellationToken.IsCancellationRequested) { return GetTagsResult.ExitCode.Canceled; }
await ProcessAndAddToMetadata(entry, cancellationToken);
}

return GetTagsResult.ExitCode.Success;

async Task ProcessAndAddToMetadata(FileEntry file, CancellationToken cancellationToken)
{
var fileRecord = new FileRecord() { FileName = file.FullPath, ModifyTime = file.ModifyTime, CreateTime = file.CreateTime, AccessTime = file.AccessTime };

var sw = new Stopwatch();
sw.Start();

if (_fileExclusionList.Any(x => file.FullPath.ToLower().Contains(x)))
{
WriteOnce.SafeLog(MsgHelp.FormatString(MsgHelp.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, fileRecord.FileName), LogLevel.Debug);
fileRecord.Status = ScanState.Skipped;
}
else
{
if (IsBinary(file.Content))
{
WriteOnce.SafeLog(MsgHelp.FormatString(MsgHelp.ID.ANALYZE_EXCLUDED_BINARY, fileRecord.FileName), LogLevel.Debug);
fileRecord.Status = ScanState.Skipped;
}
else
{
_ = _metaDataHelper?.FileExtensions.TryAdd(Path.GetExtension(file.FullPath).Replace('.', ' ').TrimStart(), 0);

LanguageInfo languageInfo = new LanguageInfo();

if (Language.FromFileName(file.FullPath, ref languageInfo))
{
_metaDataHelper?.AddLanguage(languageInfo.Name);
}
else
{
_metaDataHelper?.AddLanguage("Unknown");
languageInfo = new LanguageInfo() { Extensions = new string[] { Path.GetExtension(file.FullPath) }, Name = "Unknown" };
}

var results = await _rulesProcessor.AnalyzeFileAsync(file, languageInfo, cancellationToken, _metaDataHelper?.UniqueTags.Keys);

if (results.Any())
{
fileRecord.Status = ScanState.Affected;
fileRecord.NumFindings = results.Count;
}
else
{
fileRecord.Status = ScanState.Analyzed;
}
foreach (var matchRecord in results)
{
_metaDataHelper?.AddTagsFromMatchRecord(matchRecord);
}
}
}

sw.Stop();

fileRecord.ScanTime = sw.Elapsed;

_metaDataHelper?.Files.Add(fileRecord);
}
}


public async Task<GetTagsResult> GetResultAsync(CancellationToken cancellationToken)
{
WriteOnce.SafeLog("AnalyzeCommand::GetResultAsync", LogLevel.Trace);
WriteOnce.Operation(MsgHelp.FormatString(MsgHelp.ID.CMD_RUNNING, "Analyze"));
GetTagsResult GetTagsResult = new GetTagsResult()
{
AppVersion = Utils.GetVersionString()
};

var exitCode = await PopulateRecordsAsync(cancellationToken, _options);

//wrapup result status
if (_metaDataHelper?.Files.All(x => x.Status == ScanState.Skipped) ?? false)
{
WriteOnce.Error(MsgHelp.GetString(MsgHelp.ID.ANALYZE_NOSUPPORTED_FILETYPES));
GetTagsResult.ResultCode = GetTagsResult.ExitCode.NoMatches;
}
else if (_metaDataHelper?.UniqueTagsCount == 0)
{
WriteOnce.Error(MsgHelp.GetString(MsgHelp.ID.ANALYZE_NOPATTERNS));
GetTagsResult.ResultCode = GetTagsResult.ExitCode.NoMatches;
}
else if (_metaDataHelper != null && _metaDataHelper.Metadata != null)
{
_metaDataHelper.Metadata.DateScanned = DateScanned.ToString();
_metaDataHelper.PrepareReport();
GetTagsResult.Metadata = _metaDataHelper.Metadata; //replace instance with metadatahelper processed one
GetTagsResult.ResultCode = GetTagsResult.ExitCode.Success;
}

if (cancellationToken.IsCancellationRequested)
{
GetTagsResult.ResultCode = GetTagsResult.ExitCode.Canceled;
}

return GetTagsResult;
}

/// <summary>
/// Main entry point to start analysis from CLI; handles setting up rules, directory enumeration
/// file type detection and handoff
Expand All @@ -445,7 +574,7 @@ private bool IsBinary(Stream fileContents)
/// <returns></returns>
public GetTagsResult GetResult()
{
WriteOnce.SafeLog("GetTagsCommand::Run", LogLevel.Trace);
WriteOnce.SafeLog("GetTagsCommand::GetResult", LogLevel.Trace);
WriteOnce.Operation(MsgHelp.FormatString(MsgHelp.ID.CMD_RUNNING, "GetTags"));
GetTagsResult getTagsResult = new GetTagsResult()
{
Expand Down
2 changes: 1 addition & 1 deletion AppInspector/MetaDataHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public class MetaDataHelper

private ConcurrentDictionary<string, byte> AppTypes { get; set; } = new ConcurrentDictionary<string, byte>();
private ConcurrentDictionary<string, byte> FileNames { get; set; } = new ConcurrentDictionary<string, byte>();
private ConcurrentDictionary<string, byte> UniqueTags { get; set; } = new ConcurrentDictionary<string, byte>();
internal ConcurrentDictionary<string, byte> UniqueTags { get; set; } = new ConcurrentDictionary<string, byte>();
private ConcurrentDictionary<string, byte> Outputs { get; set; } = new ConcurrentDictionary<string, byte>();
private ConcurrentDictionary<string, byte> Targets { get; set; } = new ConcurrentDictionary<string, byte>();
private ConcurrentDictionary<string, byte> CPUTargets { get; set; } = new ConcurrentDictionary<string, byte>();
Expand Down
2 changes: 1 addition & 1 deletion AppInspector/rules/default/frameworks/java.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@
"patterns": [
{
"pattern": "googleapis",
"type": "sub-string",
"type": "substring",
"scopes": [ "code" ],
"confidence": "high"
}
Expand Down
2 changes: 1 addition & 1 deletion AppInspector/rules/default/frameworks/ruby.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
"patterns": [
{
"pattern": "google/apis",
"type": "sub-string",
"type": "substring",
"scopes": [ "code" ],
"modifiers": [ "i" ],
"confidence": "high"
Expand Down