diff --git a/Libs/Configuration/Impls/CxSASTAPIOverrides.cs b/Libs/Configuration/Impls/CxSASTAPIOverrides.cs new file mode 100644 index 00000000..583818b5 --- /dev/null +++ b/Libs/Configuration/Impls/CxSASTAPIOverrides.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CxAnalytix.Configuration.Impls +{ + public sealed class CxSASTAPIOverrides : ConfigurationElement + { + + [ConfigurationProperty("Project", IsRequired = false, DefaultValue = false)] + public bool Project + { + get => (bool)this["Project"]; + set => this["Project"] = value; + } + } +} diff --git a/Libs/Configuration/Impls/CxSASTConnection.cs b/Libs/Configuration/Impls/CxSASTConnection.cs index 6bbc1a53..5706b2a8 100644 --- a/Libs/Configuration/Impls/CxSASTConnection.cs +++ b/Libs/Configuration/Impls/CxSASTConnection.cs @@ -14,5 +14,11 @@ public String MNOUrl set { this["mnoURL"] = value; } } + [ConfigurationProperty("UseOdata", IsRequired = false)] + public CxSASTAPIOverrides Overrides + { + get => (CxSASTAPIOverrides)this["UseOdata"]; + set => this["UseOdata"] = value; + } } } diff --git a/Libs/CxRestClient/SAST/CxProjects.cs b/Libs/CxRestClient/SAST/CxProjects.cs index a863692c..b6ab0866 100644 --- a/Libs/CxRestClient/SAST/CxProjects.cs +++ b/Libs/CxRestClient/SAST/CxProjects.cs @@ -18,7 +18,11 @@ public class CxProjects { private static ILog _log = LogManager.GetLogger(typeof(CxProjects)); - private static String URL_SUFFIX = "cxrestapi/projects"; + private static readonly String REST_URL_SUFFIX = "cxrestapi/projects"; + + private static readonly int ODATA_TOP = 25; + private static readonly String ODATA_URL_SUFFIX = "cxwebinterface/odata/v1/Projects?$expand=CustomFields&$select=OwningTeamId,PresetId,Id,Name,IsPublic" + + $"&$orderby=Id asc&$top={ODATA_TOP}"; private static String _apiVersion = null; @@ -41,13 +45,32 @@ private static String GetApiVersion(CxSASTRestContext ctx, CancellationToken tok private CxProjects() { } + #region DTOs + [JsonObject(MemberSerialization.OptIn)] public class ProjectCustomFields { [JsonProperty(PropertyName = "name")] public String FieldName { get; internal set; } + + [JsonProperty(PropertyName = "FieldName")] + private String odata_FieldName + { + get => FieldName; + set => FieldName = value; + } + + [JsonProperty(PropertyName = "value")] public String FieldValue { get; internal set; } + + [JsonProperty(PropertyName = "FieldValue")] + private String odata_FieldValue + { + get => FieldValue; + set => FieldValue = value; + } + } [JsonObject(MemberSerialization.OptIn)] @@ -55,15 +78,61 @@ public class Project { [JsonProperty(PropertyName = "teamId")] public String TeamId { get; internal set; } + [JsonProperty(PropertyName = "OwningTeamId")] + private String odata_TeamId + { + get => TeamId; + set => TeamId = value; + } + + + public int PresetId { get; internal set; } + + [JsonProperty(PropertyName = "id")] public int ProjectId { get; internal set; } + [JsonProperty(PropertyName = "Id")] + private int odata_ProjectId + { + get => ProjectId; + set => ProjectId = value; + } + + [JsonProperty(PropertyName = "name")] public String ProjectName { get; internal set; } + [JsonProperty(PropertyName = "Name")] + private String odata_ProjectName + { + get => ProjectName; + set => ProjectName = value; + } + + + + [JsonProperty(PropertyName = "isPublic")] public bool IsPublic { get; internal set; } + [JsonProperty(PropertyName = "IsPublic")] + private bool odata_IsPublic + { + get => IsPublic; + set => IsPublic = value; + } + + + [JsonProperty(PropertyName = "customFields")] public List CustomFields { get; internal set; } + [JsonProperty(PropertyName = "CustomFields")] + private List odata_CustomFields + { + get => CustomFields; + set => CustomFields = value; + } + + [JsonProperty(PropertyName = "isBranched")] public bool IsBranched { get; internal set; } @@ -78,6 +147,7 @@ public class Project public override string ToString() => $"{ProjectId}:{ProjectName} [TeamId: {TeamId} Public: {IsPublic} CustomFields: {CustomFields.Count}]"; } + #endregion private class ProjectReader : IEnumerable, IEnumerator, IDisposable { @@ -161,12 +231,57 @@ public void Reset() { throw new NotImplementedException(); } + } + private static IEnumerable GetProjects_odata(CxSASTRestContext ctx, CancellationToken token) + { + List returnedResults = new(); + + var filter = new Dictionary(); + List fetchedPage = null; + + do + { + String requestUrl = UrlUtils.MakeUrl(ctx.Sast.ApiUrl, ODATA_URL_SUFFIX, filter); + using (var projectReader = WebOperation.ExecuteGet( + ctx.Sast.Json.CreateClient + , (response) => + { + using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result)) + using (var jtr = new JsonTextReader(sr)) + { + JToken jt = JToken.Load(jtr); + + return new ProjectReader(jt["value"], ctx, token); + } + } + , requestUrl + , ctx.Sast + , token)) + fetchedPage = new List(projectReader); + + if (fetchedPage != null) + { + returnedResults.AddRange(fetchedPage); + filter["$filter"] = $"id gt {fetchedPage[fetchedPage.Count - 1].ProjectId}"; + } + + + } while (fetchedPage != null && fetchedPage.Count == ODATA_TOP); + + return returnedResults; } + public static IEnumerable GetProjects(CxSASTRestContext ctx, CancellationToken token, bool useOData) + { + if (useOData) + return GetProjects_odata(ctx, token); + else + return GetProjects_rest(ctx, token); + } - public static IEnumerable GetProjects(CxSASTRestContext ctx, CancellationToken token) + private static IEnumerable GetProjects_rest(CxSASTRestContext ctx, CancellationToken token) { using (var projectReader = WebOperation.ExecuteGet( ctx.Sast.Json.CreateClient @@ -180,7 +295,7 @@ public static IEnumerable GetProjects(CxSASTRestContext ctx, Cancellati return new ProjectReader(jt, ctx, token); } } - , UrlUtils.MakeUrl(ctx.Sast.ApiUrl, URL_SUFFIX) + , UrlUtils.MakeUrl(ctx.Sast.ApiUrl, REST_URL_SUFFIX) , ctx.Sast , token, apiVersion: GetApiVersion(ctx, token) )) return new List(projectReader); diff --git a/Libs/CxRestClient/SAST/CxScanStatistics.cs b/Libs/CxRestClient/SAST/CxScanStatistics.cs index 140c0541..988eee37 100644 --- a/Libs/CxRestClient/SAST/CxScanStatistics.cs +++ b/Libs/CxRestClient/SAST/CxScanStatistics.cs @@ -1,4 +1,5 @@ using CxAnalytix.Exceptions; +using CxAnalytix.Extensions; using CxRestClient.Utility; using log4net; using Newtonsoft.Json; @@ -7,6 +8,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Linq.Expressions; using System.Net; using System.Net.Http; using System.Text; @@ -270,24 +272,51 @@ internal FullScanStatistics() public static FullScanStatistics GetScanFullStatistics(CxSASTRestContext ctx, CancellationToken token, String scanId) { - var statistics = GetScanStatistics(ctx, token, scanId); - - var pf = GetScanParsedFiles(ctx, token, scanId); - var fq = GetScanFailedQueries(ctx, token, scanId); - var fgq = GetScanFailedGeneralQueries(ctx, token, scanId); - var sgq = GetScanSuccessfulGeneralQueries(ctx, token, scanId); - - if (statistics.Result == null) - return null; - - return new FullScanStatistics() - { - Statistics = statistics.Result, - ParsedFiles = pf.Result, - FailedQueries = fq.Result, - FailedGeneralQueries = fgq.Result, - SuccessGeneralQueries = sgq.Result - }; + CancellationTokenSource localToken = new(); + + List runningTasks = new(); + + using (token.Register(() => localToken.Cancel())) + try + { + var statistics = GetScanStatistics(ctx, localToken.Token, scanId); + runningTasks.Add(statistics); + + var pf = GetScanParsedFiles(ctx, localToken.Token, scanId); + runningTasks.Add(pf); + + var fq = GetScanFailedQueries(ctx, localToken.Token, scanId); + runningTasks.Add(fq); + + var fgq = GetScanFailedGeneralQueries(ctx, localToken.Token, scanId); + runningTasks.Add(fgq); + + var sgq = GetScanSuccessfulGeneralQueries(ctx, localToken.Token, scanId); + runningTasks.Add(sgq); + + + if (statistics.Result != null) + return new FullScanStatistics() + { + Statistics = statistics.Result, + ParsedFiles = pf.Result, + FailedQueries = fq.Result, + FailedGeneralQueries = fgq.Result, + SuccessGeneralQueries = sgq.Result + }; + } + catch (Exception) + { + localToken.Cancel(); + throw; + } + finally + { + runningTasks.SafeWaitAllToEnd(); + runningTasks.DisposeTasks(); + } + + return null; } } diff --git a/Libs/Exceptions/ScanCrawlException.cs b/Libs/Exceptions/ScanCrawlException.cs new file mode 100644 index 00000000..8810557e --- /dev/null +++ b/Libs/Exceptions/ScanCrawlException.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CxAnalytix.Exceptions +{ + public class ScanCrawlException : Exception + { + private static String MakeMessage(String scanId, String projectName, String teamName) => + $"Exception caught processing scan [{scanId}] in [{projectName}]" + + (!String.IsNullOrEmpty(teamName) ? $" assigned to team(s) [{teamName}]" : ""); + + + public ScanCrawlException (String scanId, String projectName, String teamName) + : base(ScanCrawlException.MakeMessage(scanId, projectName, teamName) ) + { + + } + public ScanCrawlException(String scanId, String projectName, String teamName, Exception ex) + : base(ScanCrawlException.MakeMessage(scanId, projectName, teamName), ex) + { + + } + } +} diff --git a/Libs/Executive/ExecuteLoop.cs b/Libs/Executive/ExecuteLoop.cs index 7770c0b1..89f98777 100644 --- a/Libs/Executive/ExecuteLoop.cs +++ b/Libs/Executive/ExecuteLoop.cs @@ -11,7 +11,7 @@ namespace CxAnalytix.Executive { public class ExecuteLoop : ExecuteOnce { - private static readonly ILog appLog = LogManager.GetLogger(typeof(ExecuteLoop)); + private static readonly ILog _log = LogManager.GetLogger(typeof(ExecuteLoop)); public static new void Execute(CancellationTokenSource t) { @@ -35,19 +35,23 @@ public static new void Execute(CancellationTokenSource t) } catch (Exception ex) { - appLog.Error("Vulnerability data transformation aborted due to unhandled exception.", ex); + _log.Error("Vulnerability data transformation aborted due to unhandled exception.", ex); } GC.Collect(); - Task.Delay(Service.ProcessPeriodMinutes * 60 * 1000, t.Token).Wait(); + using (var delay = Task.Delay(Service.ProcessPeriodMinutes * 60 * 1000, t.Token)) + delay.Wait(t.Token); + } while (!t.Token.IsCancellationRequested); + _log.Info("Execution complete, ending."); + } private static void Fatal(Exception ex, CancellationTokenSource ct) { - appLog.Error("Fatal exception caught, program ending.", ex); + _log.Error("Fatal exception caught, program ending.", ex); ct.Cancel(); Process.GetCurrentProcess().Kill(true); diff --git a/Libs/Executive/ExecuteOnce.cs b/Libs/Executive/ExecuteOnce.cs index 67d8167e..6f5cdcca 100644 --- a/Libs/Executive/ExecuteOnce.cs +++ b/Libs/Executive/ExecuteOnce.cs @@ -64,6 +64,16 @@ private static IEnumerable LoadTransformers(ILifetimeScope lifeSco return retVal; } + private static void LogExceptionAsError(int counter, Exception ex) + { + if (ex == null) + return; + + _log.Error($"Exception Level {counter}", ex); + + LogExceptionAsError(counter++, ex.InnerException); + } + public static void Execute(CancellationTokenSource? t = null) { using (var scope = _xformersContainer.BeginLifetimeScope()) @@ -93,6 +103,22 @@ public static void Execute(CancellationTokenSource? t = null) { xformer.DoTransform(t.Token); } + catch (AggregateException aex) + { + _log.Error($"Unhandled exception when executing transformer module: {xformer.DisplayName}", aex); + int counter = 0; + + + foreach(var ex in aex.InnerExceptions) + { + _log.Error($"-- BEGIN AGGREGATE EXCEPTION {++counter} --"); + + LogExceptionAsError(0, ex); + + _log.Error($"-- END AGGREGATE EXCEPTION {counter} --"); + } + + } catch (Exception ex) { _log.Error($"Unhandled exception when executing transformer module: {xformer.DisplayName}", ex); diff --git a/Libs/Extensions/Threading.cs b/Libs/Extensions/Threading.cs index 222df7e9..992d8449 100644 --- a/Libs/Extensions/Threading.cs +++ b/Libs/Extensions/Threading.cs @@ -11,7 +11,7 @@ public static class Threading { private static readonly int WAIT_MAX = 120000; - public static Task DisposeTask(this Task task) + private static void doDispose(Task task) { if (task != null) { @@ -19,8 +19,70 @@ public static Task DisposeTask(this Task task) if (task.IsCompleted) task.Dispose(); } + } + + public static Task DisposeTask(this Task task) + { + doDispose(task); + return null; + } + public static Task DisposeTask(this Task task) + { + doDispose(task); return null; } + public static void DisposeTasks(this IEnumerable tasks) + { + foreach(var task in tasks) + doDispose(task); + } + + public static void SafeWaitToEnd(this Task task) + { + if (task == null) + return; + + try + { + if (!task.IsCompleted) + task.Wait(); + } + // Eat the exceptions so it ensures all the tasks end. + catch (AggregateException) + { + } + catch (TaskCanceledException) + { + } + } + + + public static void SafeWaitAllToEnd(this IEnumerable tasks) + { + if (tasks == null) + return; + + foreach (var task in tasks) + { + if (task == null) + continue; + + try + { + if (!task.IsCompleted) + task.Wait(); + } + // Eat the exceptions so it ensures all the tasks end. + catch (AggregateException) + { + } + catch (TaskCanceledException) + { + } + } + + } + } } diff --git a/XForm/CxOneTransformer/Transformer.cs b/XForm/CxOneTransformer/Transformer.cs index 8d011d93..3d460ae1 100644 --- a/XForm/CxOneTransformer/Transformer.cs +++ b/XForm/CxOneTransformer/Transformer.cs @@ -1,4 +1,5 @@ using CxAnalytix.Configuration.Impls; +using CxAnalytix.Exceptions; using CxAnalytix.Extensions; using CxAnalytix.Interfaces.Outputs; using CxAnalytix.XForm.Common; @@ -10,10 +11,14 @@ using SDK.Modules.Transformer.Data; using System; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Text; using System.Web; using static CxRestClient.CXONE.CxConfiguration; +using static CxRestClient.CXONE.CxGroups; using static CxRestClient.CXONE.CxSastPredicates; +using static CxRestClient.CXONE.CxSastScanMetadata; +using static CxRestClient.CXONE.CxScanResults; using static CxRestClient.SCA.CxRiskState; using static SDK.Modules.Transformer.Data.ScanDescriptor; using CxOneConnection = CxAnalytix.XForm.CxOneTransformer.Config.CxOneConnection; @@ -94,7 +99,7 @@ private void TrackScanEngineStats(String projectId, CxScans.Scan scan) } - public override void DoTransform(CancellationToken token) + public override void DoTransform(CancellationToken outerToken) { var creds = Configuration.Impls.Config.GetConfig(); @@ -110,154 +115,229 @@ public override void DoTransform(CancellationToken token) Context = restBuilder.Build(); - QueryData = new(Context, token); - Predicates = new(Context, token); + CancellationTokenSource localToken = new(); - ThreadOpts.CancellationToken = token; + using (outerToken.Register(() => localToken?.Cancel())) + { - _log.Debug("Starting CxOne transform"); + QueryData = new(Context, localToken.Token); + Predicates = new(Context, localToken.Token); - ProjectsFetchTask = CxProjects.GetProjects(Context, token); - ApplicationsFetchTask = CxApplications.GetApplications(Context, token); + ThreadOpts.CancellationToken = localToken.Token; - using (var groupsTask = CxGroups.GetGroups(Context, token)) - using (var lastScansTask = CxProjects.GetProjectLatestCompletedScans(Context, token)) - { + _log.Debug("Starting CxOne transform"); + + ProjectsFetchTask = CxProjects.GetProjects(Context, localToken.Token); + ApplicationsFetchTask = CxApplications.GetApplications(Context, localToken.Token); - _log.Debug("Resolving projects."); - var projectDescriptors = new ConcurrentBag(); + Task>? groupsTask = null; + Task>? lastScansTask = null; + Task scanMetadata = null; - Parallel.ForEach(ProjectsFetchTask.Result, ThreadOpts, (p) => + List sweepTasks = new(); + + try { - ProjectConfigFetchTasks.TryAdd(p.ProjectId, CxConfiguration.GetProjectConfiguration(Context, ThreadOpts.CancellationToken, p.ProjectId)); - String groupsString = String.Join(",", p.Groups.ConvertAll((groupId) => groupsTask.Result[groupId].Path)); - // Projects don't need to have a team assignment, unlike in SAST - if (!((p.Groups.Count == 0) ? Filter.Matches("", p.ProjectName) - : p.Groups.Any((t) => Filter.Matches(groupsTask.Result[t].Path, p.ProjectName)))) - { - if (_log.IsDebugEnabled) - _log.Debug($"FILTERED: Project: [{p.ProjectName}] with assigned groups " + - $"[{groupsString}]"); - return; - } - else - { - projectDescriptors.Add(new ProjectDescriptor() - { - ProjectId = p.ProjectId, - ProjectName = p.ProjectName, - TeamName = groupsString, - PresetName = ProjectConfigFetchTasks[p.ProjectId].Result.Preset + sweepTasks.Add(groupsTask = CxGroups.GetGroups(Context, localToken.Token)); + sweepTasks.Add(lastScansTask = CxProjects.GetProjectLatestCompletedScans(Context, localToken.Token)); - }); - } - }); + _log.Debug("Resolving projects."); - State.ConfirmProjects(projectDescriptors); + State.ConfirmProjects(ResolveProjects(groupsTask)); - _log.Info($"{State.ProjectCount} projects are targets to check for new scans. Since last crawl: {State.DeletedProjects}" - + $" projects removed, {State.NewProjects} new projects."); + _log.Info($"{State.ProjectCount} projects are targets to check for new scans. Since last crawl: {State.DeletedProjects}" + + $" projects removed, {State.NewProjects} new projects."); - _log.Debug("Resolving scans."); + _log.Debug("Resolving scans."); - var loadedScans = new CxScans.ScanIndex(); + var loadedScans = ResolveScans(lastScansTask); + _log.Info($"Crawling {State.ScanCount} scans."); - Parallel.ForEach(State.Projects, ThreadOpts, (projDescriptor) => - { - if (!lastScansTask.Result.ContainsKey(projDescriptor.ProjectId)) + sweepTasks.Add(scanMetadata = CxSastScanMetadata.GetScanMetadata(Context, localToken.Token, ThreadOpts, State.ScopeScanIds)); + + Parallel.ForEach(State.Projects, ThreadOpts, + (project) => { - _log.Info($"Project {projDescriptor.ProjectId}:{projDescriptor.TeamName}:{projDescriptor.ProjectName} contains no scans."); - return; - } + if (State.GetScanCountForProject(project.ProjectId) <= 0) + return; - var latestScanDateForProject = lastScansTask.Result[projDescriptor.ProjectId].Completed; + CancellationTokenSource loopToken = new(); + Task sca_risk_states = null; - // This skips some API I/O since we know the last scan date of some projects. - if (projDescriptor.LastScanCheckDate.CompareTo(latestScanDateForProject) < 0) - { - using (var scanCollection = CxScans.GetCompletedScans(Context, ThreadOpts.CancellationToken, projDescriptor.ProjectId)) - { - foreach (var s in scanCollection.Result.Scans) + using (ThreadOpts.CancellationToken.Register(() => loopToken?.Cancel())) + try { - // Add to crawl state. - if (_log.IsTraceEnabled()) - _log.Trace($"CxOne scan record: {s}"); + sca_risk_states = CxScanResults.GetScaRiskStates(Context, loopToken.Token, project.ProjectId); - State.AddScan(Convert.ToString(s.ProjectId), s.ScanType, ScanProductType.CXONE, s.ScanId, s.Updated, s.EnginesAsString); - TrackScanEngineStats(projDescriptor.ProjectId, s); - } + using (var pinfoTrx = Output.StartTransaction()) + { + OutputProjectInfoRecords(pinfoTrx, project); - loadedScans.SyncCombine(scanCollection.Result.Scans); - } - } - else - _log.Info($"Project {projDescriptor.ProjectId}:{projDescriptor.TeamName}:{projDescriptor.ProjectName} has no new scans to process."); - }); + if (!loopToken.IsCancellationRequested) + pinfoTrx.Commit(); + } - if (State.Projects == null) + + foreach (var scan in State.GetScansForProject(project.ProjectId)) + { + if (loopToken.IsCancellationRequested) + break; + + + using (var scanTrx = Output.StartTransaction()) + { + var projid = (scan.Project == null) ? "Unknown ProjectId" : scan.Project.ProjectId; + var projname = (scan.Project == null) ? "Unknown Project Name" : scan.Project.ProjectName; + + _log.Info($"Processing {scan.ScanProduct} scan [{scan.ScanId}] for project [{projname}]" + + $" assigned to team(s) [{scan.Project.TeamName}]" + + $" finished [{scan.FinishedStamp}]"); + + Task rpt = null; + rpt = CxScanResults.GetScanResults(Context, loopToken.Token, scan.ScanId); + + try + { + if (rpt.Result.SastResults != null && rpt.Result.SastResults.Count > 0) + OutputSastScanResults(scanTrx, project, scan, loadedScans, rpt.Result.SastResults, scanMetadata.Result[scan.ScanId]); + + if (rpt.Result.ScaResults != null && rpt.Result.ScaResults.Count > 0) + OutputScaScanResults(scanTrx, project, scan, loadedScans, rpt.Result.ScaResults, sca_risk_states.Result); + + if (!loopToken.IsCancellationRequested && scanTrx.Commit()) + State.ScanCompleted(scan); + } + catch (Exception ex) + { + if (loopToken != null) + loopToken.Cancel(); + + throw new ScanCrawlException(scan.ScanId, project.ProjectName, project.TeamName, ex); + } + finally + { + if (rpt != null) + { + rpt.SafeWaitToEnd(); + rpt.DisposeTask(); + } + } + } + } + } + catch (ScanCrawlException uex) + { + loopToken.Cancel(); + _log.Error($"Scan crawling stopped for project id [{project.ProjectId}].", uex); + } + catch (Exception) + { + loopToken.Cancel(); + throw; + } + finally + { + + if (sca_risk_states != null) + { + sca_risk_states.SafeWaitToEnd(); + sca_risk_states.DisposeTask(); + } + } + }); + } + catch (Exception) { - _log.Error("Scans to crawl do not appear to be resolved, unable to crawl scan data."); - return; + localToken.Cancel(); + throw; } + finally + { + sweepTasks.SafeWaitAllToEnd(); + sweepTasks.DisposeTasks(); + } + } + } - _log.Info($"Crawling {State.ScanCount} scans."); + private CxScans.ScanIndex ResolveScans(Task> lastScansTask) + { + var loadedScans = new CxScans.ScanIndex(); + Parallel.ForEach(State.Projects, ThreadOpts, (projDescriptor) => + { + if (!lastScansTask.Result.ContainsKey(projDescriptor.ProjectId)) + { + _log.Info($"Project {projDescriptor.ProjectId}:{projDescriptor.TeamName}:{projDescriptor.ProjectName} contains no scans."); + return; + } - var scanMetadata = CxSastScanMetadata.GetScanMetadata(Context, token, ThreadOpts, State.ScopeScanIds); + var latestScanDateForProject = lastScansTask.Result[projDescriptor.ProjectId].Completed; - Parallel.ForEach(State.Projects, ThreadOpts, - (project) => + // This skips some API I/O since we know the last scan date of some projects. + if (projDescriptor.LastScanCheckDate.CompareTo(latestScanDateForProject) < 0) { - if (State.GetScanCountForProject(project.ProjectId) <= 0) - return; - - using (var sca_risk_states = CxScanResults.GetScaRiskStates(Context, ThreadOpts.CancellationToken, project.ProjectId)) + using (var scanCollection = CxScans.GetCompletedScans(Context, ThreadOpts.CancellationToken, projDescriptor.ProjectId)) { - - using (var pinfoTrx = Output.StartTransaction()) + foreach (var s in scanCollection.Result.Scans) { - OutputProjectInfoRecords(pinfoTrx, project); + // Add to crawl state. + if (_log.IsTraceEnabled()) + _log.Trace($"CxOne scan record: {s}"); - if (!ThreadOpts.CancellationToken.IsCancellationRequested) - pinfoTrx.Commit(); + State.AddScan(Convert.ToString(s.ProjectId), s.ScanType, ScanProductType.CXONE, s.ScanId, s.Updated, s.EnginesAsString); + TrackScanEngineStats(projDescriptor.ProjectId, s); } + loadedScans.SyncCombine(scanCollection.Result.Scans); + } + } + else + _log.Info($"Project {projDescriptor.ProjectId}:{projDescriptor.TeamName}:{projDescriptor.ProjectName} has no new scans to process."); + }); - foreach (var scan in State.GetScansForProject(project.ProjectId)) - { - if (ThreadOpts.CancellationToken.IsCancellationRequested) - break; + if (State.Projects == null) + throw new UnrecoverableOperationException ("Scans to crawl do not appear to be resolved, unable to crawl scan data."); + return loadedScans; + } - using (var scanTrx = Output.StartTransaction()) - { - var projid = (scan.Project == null) ? "Unknown ProjectId" : scan.Project.ProjectId; - var projname = (scan.Project == null) ? "Unknown Project Name" : scan.Project.ProjectName; - _log.Info($"Processing {scan.ScanProduct} scan {scan.ScanId}:{projid}:{projname}[{scan.FinishedStamp}]"); - using (var rpt = CxScanResults.GetScanResults(Context, ThreadOpts.CancellationToken, scan.ScanId)) - { - if (rpt.Result.SastResults != null && rpt.Result.SastResults.Count > 0) - OutputSastScanResults(scanTrx, project, scan, loadedScans, rpt.Result.SastResults, scanMetadata.Result[scan.ScanId]); + private ConcurrentBag ResolveProjects(Task>? groupsTask) + { + var projectDescriptors = new ConcurrentBag(); - if (rpt.Result.ScaResults != null && rpt.Result.ScaResults.Count > 0) - OutputScaScanResults(scanTrx, project, scan, loadedScans, rpt.Result.ScaResults, sca_risk_states.Result); - else - sca_risk_states.Wait(); + Parallel.ForEach(ProjectsFetchTask.Result, ThreadOpts, (p) => + { + ProjectConfigFetchTasks.TryAdd(p.ProjectId, CxConfiguration.GetProjectConfiguration(Context, ThreadOpts.CancellationToken, p.ProjectId)); + String groupsString = String.Join(",", p.Groups.ConvertAll((groupId) => groupsTask.Result[groupId].Path)); - if (!ThreadOpts.CancellationToken.IsCancellationRequested && scanTrx.Commit()) - State.ScanCompleted(scan); - } - } - } - } - }); - } + // Projects don't need to have a team assignment, unlike in SAST + if (!((p.Groups.Count == 0) ? Filter.Matches("", p.ProjectName) + : p.Groups.Any((t) => Filter.Matches(groupsTask.Result[t].Path, p.ProjectName)))) + { + if (_log.IsDebugEnabled) + _log.Debug($"FILTERED: Project: [{p.ProjectName}] with assigned groups " + + $"[{groupsString}]"); + return; + } + else + { + projectDescriptors.Add(new ProjectDescriptor() + { + ProjectId = p.ProjectId, + ProjectName = p.ProjectName, + TeamName = groupsString, + PresetName = ProjectConfigFetchTasks[p.ProjectId].Result.Preset + + }); + } + }); + return projectDescriptors; } protected override void AddAdditionalProjectInfo(IDictionary here, string projectId) @@ -546,12 +626,19 @@ public override void Dispose() if (Predicates != null) Predicates.Dispose(); - ProjectsFetchTask = ProjectsFetchTask.DisposeTask(); - ApplicationsFetchTask = ApplicationsFetchTask.DisposeTask(); + ProjectsFetchTask.SafeWaitToEnd(); foreach (var key in ProjectConfigFetchTasks.Keys) + { + ProjectConfigFetchTasks[key].SafeWaitToEnd(); ProjectConfigFetchTasks[key] = ProjectConfigFetchTasks[key].DisposeTask(); - ProjectConfigFetchTasks = null; + } + + ProjectsFetchTask = ProjectsFetchTask.DisposeTask(); + + ApplicationsFetchTask.SafeWaitToEnd(); + ApplicationsFetchTask = ApplicationsFetchTask.DisposeTask(); + } } diff --git a/XForm/SastTransformer/Transformer.cs b/XForm/SastTransformer/Transformer.cs index 7de7d560..fb2b8f59 100644 --- a/XForm/SastTransformer/Transformer.cs +++ b/XForm/SastTransformer/Transformer.cs @@ -326,7 +326,8 @@ private async Task ResolveScans() _sastVersionTask = GetSASTVersion(); - var projectsTask = Task.Run(() => CxProjects.GetProjects(RestContext, ThreadOpts.CancellationToken), ThreadOpts.CancellationToken); + var projectsTask = Task.Run(() => CxProjects.GetProjects(RestContext, ThreadOpts.CancellationToken, + Config.GetConfig().Overrides.Project), ThreadOpts.CancellationToken); Policies = await policyTask; Teams = await teamsTask; diff --git a/manual/configuration-general.tex b/manual/configuration-general.tex index 69ae3cc4..6317898b 100644 --- a/manual/configuration-general.tex +++ b/manual/configuration-general.tex @@ -71,8 +71,9 @@ \subsubsection{Checkmarx SAST Connection Configuration} mnoURL="" TimeoutSeconds="" ValidateCertificates="true" - RetryLoop="" - /> + RetryLoop=""> + + \end{xml} \begin{table}[h] @@ -94,6 +95,32 @@ \subsubsection{Checkmarx SAST Connection Configuration} \end{tabularx} \end{table} +\noindent\\The sub-element \texttt{UseOdata} is optional. The properties for the element indicate +when some API I/O should be done using the OData API instead of the REST API. This may be used +in cases where REST APIs are performing poorly due to the size of the response payload. +\footnote{It is recommended to avoid using the OData API unless there are no other options.} + +\noindent\\Note that using the Odata API introduces the following limitations: + +\begin{itemize} + \item Project + \begin{itemize} + \item Branch projects no longer export any branch information. The branch information is + not available via the Odata API. + \end{itemize} +\end{itemize} + +\begin{table}[h] + \caption{UseOdata Attributes} + \begin{tabularx}{\textwidth}{cccl} + \toprule + \textbf{Attribute} & \textbf{Default} & \textbf{Required} & \textbf{Description}\\ + \midrule + \texttt{Project} & False & No & \makecell[l]{Use the Odata API to retrieve SAST projects.}\\ + \bottomrule + \end{tabularx} +\end{table} + \begin{xml}{CxSASTCredentials}{\expandsenv\encrypts}{} Project \& Scans->Save Sast Scan \item Reports->Generate Scan Reports + \item Scan Results->View Results + \end{itemize} + + \item If you have \hyperref[sec:connection]{configured any APIs to use OData}, + the CxAnalytix role should have the following additional permissions: + \begin{itemize} + \item API->Use Odata \end{itemize} \end{itemize} @@ -142,7 +149,7 @@ \subsection{SCA} \begin{itemize} \item The service account should be assigned at a team level that allows visibility to all projects that require crawling. Usually this is the \verb|/CxServer| - team but will depend on your configured team heirarchy. Any projects assigned to teams above or at a sibling level of the service account's assigned team + team but will depend on your configured team hierarchy. Any projects assigned to teams above or at a sibling level of the service account's assigned team will not be visible to crawling requests. \item A role named CxAnalytix should be created and assigned to the service account user. The role should have the following minimum permissions: diff --git a/manual/release_notes-content.tex b/manual/release_notes-content.tex index 22abb5dc..1c54166c 100644 --- a/manual/release_notes-content.tex +++ b/manual/release_notes-content.tex @@ -1,3 +1,25 @@ +\section{2.1.2} + +\subsection*{FEATURES} + \begin{itemize} + \item CheckmarxOne crawl stabilization. + \item SAST scan statistic record retrieval stabilization. + \end{itemize} + + +\subsection*{UPDATES} +\begin{itemize} + \item Issue 225 - Documentation updates for required SAST user roles. + \\\\Additional roles are required to allow retrieval of the scan statistics data. + + \item The OData API can be used in lieu of the REST API for some scenarios where the + REST API performs poorly. + \\\\Please review the configuration documentation sections to understand any limitations + introduced by using the OData API. Additional security roles may be needed for + your service account to be able to use the OData API; please review the deployment + documentation to understand any required security role changes. +\end{itemize} + \section{2.1.1} \subsection*{FEATURES}