Skip to content

Commit

Permalink
Feature/stability (#226)
Browse files Browse the repository at this point in the history 
* cxone crawl stability

* fix issue #225

* scan stat record stabilization

* documentation updates

* enable odata api
  • Loading branch information
@nleach999
nleach999 committed Aug 4, 2023
1 parent bf4afe3 commit 37e85fd
Show file tree
Hide file tree
Showing 13 changed files with 572 additions and 139 deletions.
20 changes: 20 additions & 0 deletions Libs/Configuration/Impls/CxSASTAPIOverrides.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace CxAnalytix.Configuration.Impls
{
public sealed class CxSASTAPIOverrides : ConfigurationElement
{

[ConfigurationProperty("Project", IsRequired = false, DefaultValue = false)]
public bool Project
{
get => (bool)this["Project"];
set => this["Project"] = value;
}
}
}
6 changes: 6 additions & 0 deletions Libs/Configuration/Impls/CxSASTConnection.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,11 @@ public String MNOUrl
set { this["mnoURL"] = value; }
}

[ConfigurationProperty("UseOdata", IsRequired = false)]
public CxSASTAPIOverrides Overrides
{
get => (CxSASTAPIOverrides)this["UseOdata"];
set => this["UseOdata"] = value;
}
}
}
121 changes: 118 additions & 3 deletions Libs/CxRestClient/SAST/CxProjects.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@ public class CxProjects
{
private static ILog _log = LogManager.GetLogger(typeof(CxProjects));

private static String URL_SUFFIX = "cxrestapi/projects";
private static readonly String REST_URL_SUFFIX = "cxrestapi/projects";

private static readonly int ODATA_TOP = 25;
private static readonly String ODATA_URL_SUFFIX = "cxwebinterface/odata/v1/Projects?$expand=CustomFields&$select=OwningTeamId,PresetId,Id,Name,IsPublic" +
$"&$orderby=Id asc&$top={ODATA_TOP}";

private static String _apiVersion = null;

Expand All @@ -41,29 +45,94 @@ private static String GetApiVersion(CxSASTRestContext ctx, CancellationToken tok
private CxProjects()
{ }

#region DTOs

[JsonObject(MemberSerialization.OptIn)]
public class ProjectCustomFields
{
[JsonProperty(PropertyName = "name")]
public String FieldName { get; internal set; }

[JsonProperty(PropertyName = "FieldName")]
private String odata_FieldName
{
get => FieldName;
set => FieldName = value;
}


[JsonProperty(PropertyName = "value")]
public String FieldValue { get; internal set; }

[JsonProperty(PropertyName = "FieldValue")]
private String odata_FieldValue
{
get => FieldValue;
set => FieldValue = value;
}

}

[JsonObject(MemberSerialization.OptIn)]
public class Project
{
[JsonProperty(PropertyName = "teamId")]
public String TeamId { get; internal set; }
[JsonProperty(PropertyName = "OwningTeamId")]
private String odata_TeamId
{
get => TeamId;
set => TeamId = value;
}



public int PresetId { get; internal set; }


[JsonProperty(PropertyName = "id")]
public int ProjectId { get; internal set; }
[JsonProperty(PropertyName = "Id")]
private int odata_ProjectId
{
get => ProjectId;
set => ProjectId = value;
}


[JsonProperty(PropertyName = "name")]
public String ProjectName { get; internal set; }
[JsonProperty(PropertyName = "Name")]
private String odata_ProjectName
{
get => ProjectName;
set => ProjectName = value;
}




[JsonProperty(PropertyName = "isPublic")]
public bool IsPublic { get; internal set; }
[JsonProperty(PropertyName = "IsPublic")]
private bool odata_IsPublic
{
get => IsPublic;
set => IsPublic = value;
}



[JsonProperty(PropertyName = "customFields")]
public List<ProjectCustomFields> CustomFields { get; internal set; }
[JsonProperty(PropertyName = "CustomFields")]
private List<ProjectCustomFields> odata_CustomFields
{
get => CustomFields;
set => CustomFields = value;
}



[JsonProperty(PropertyName = "isBranched")]
public bool IsBranched { get; internal set; }
Expand All @@ -78,6 +147,7 @@ public class Project
public override string ToString() =>
$"{ProjectId}:{ProjectName} [TeamId: {TeamId} Public: {IsPublic} CustomFields: {CustomFields.Count}]";
}
#endregion

private class ProjectReader : IEnumerable<Project>, IEnumerator<Project>, IDisposable
{
Expand Down Expand Up @@ -161,12 +231,57 @@ public void Reset()
{
throw new NotImplementedException();
}
}

private static IEnumerable<Project> GetProjects_odata(CxSASTRestContext ctx, CancellationToken token)
{
List<Project> returnedResults = new();

var filter = new Dictionary<String, String>();
List<Project> fetchedPage = null;

do
{
String requestUrl = UrlUtils.MakeUrl(ctx.Sast.ApiUrl, ODATA_URL_SUFFIX, filter);

using (var projectReader = WebOperation.ExecuteGet<ProjectReader>(
ctx.Sast.Json.CreateClient
, (response) =>
{
using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result))
using (var jtr = new JsonTextReader(sr))
{
JToken jt = JToken.Load(jtr);
return new ProjectReader(jt["value"], ctx, token);
}
}
, requestUrl
, ctx.Sast
, token))
fetchedPage = new List<Project>(projectReader);

if (fetchedPage != null)
{
returnedResults.AddRange(fetchedPage);
filter["$filter"] = $"id gt {fetchedPage[fetchedPage.Count - 1].ProjectId}";
}


} while (fetchedPage != null && fetchedPage.Count == ODATA_TOP);

return returnedResults;
}

public static IEnumerable<Project> GetProjects(CxSASTRestContext ctx, CancellationToken token, bool useOData)
{
if (useOData)
return GetProjects_odata(ctx, token);
else
return GetProjects_rest(ctx, token);
}

public static IEnumerable<Project> GetProjects(CxSASTRestContext ctx, CancellationToken token)
private static IEnumerable<Project> GetProjects_rest(CxSASTRestContext ctx, CancellationToken token)
{
using (var projectReader = WebOperation.ExecuteGet<ProjectReader>(
ctx.Sast.Json.CreateClient
Expand All @@ -180,7 +295,7 @@ public static IEnumerable<Project> GetProjects(CxSASTRestContext ctx, Cancellati
return new ProjectReader(jt, ctx, token);
}
}
, UrlUtils.MakeUrl(ctx.Sast.ApiUrl, URL_SUFFIX)
, UrlUtils.MakeUrl(ctx.Sast.ApiUrl, REST_URL_SUFFIX)
, ctx.Sast
, token, apiVersion: GetApiVersion(ctx, token) ))
return new List<Project>(projectReader);
Expand Down
65 changes: 47 additions & 18 deletions Libs/CxRestClient/SAST/CxScanStatistics.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using CxAnalytix.Exceptions;
using CxAnalytix.Extensions;
using CxRestClient.Utility;
using log4net;
using Newtonsoft.Json;
Expand All @@ -7,6 +8,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Net;
using System.Net.Http;
using System.Text;
Expand Down Expand Up @@ -270,24 +272,51 @@ internal FullScanStatistics()

public static FullScanStatistics GetScanFullStatistics(CxSASTRestContext ctx, CancellationToken token, String scanId)
{
var statistics = GetScanStatistics(ctx, token, scanId);

var pf = GetScanParsedFiles(ctx, token, scanId);
var fq = GetScanFailedQueries(ctx, token, scanId);
var fgq = GetScanFailedGeneralQueries(ctx, token, scanId);
var sgq = GetScanSuccessfulGeneralQueries(ctx, token, scanId);

if (statistics.Result == null)
return null;

return new FullScanStatistics()
{
Statistics = statistics.Result,
ParsedFiles = pf.Result,
FailedQueries = fq.Result,
FailedGeneralQueries = fgq.Result,
SuccessGeneralQueries = sgq.Result
};
CancellationTokenSource localToken = new();

List<Task> runningTasks = new();

using (token.Register(() => localToken.Cancel()))
try
{
var statistics = GetScanStatistics(ctx, localToken.Token, scanId);
runningTasks.Add(statistics);

var pf = GetScanParsedFiles(ctx, localToken.Token, scanId);
runningTasks.Add(pf);

var fq = GetScanFailedQueries(ctx, localToken.Token, scanId);
runningTasks.Add(fq);

var fgq = GetScanFailedGeneralQueries(ctx, localToken.Token, scanId);
runningTasks.Add(fgq);

var sgq = GetScanSuccessfulGeneralQueries(ctx, localToken.Token, scanId);
runningTasks.Add(sgq);


if (statistics.Result != null)
return new FullScanStatistics()
{
Statistics = statistics.Result,
ParsedFiles = pf.Result,
FailedQueries = fq.Result,
FailedGeneralQueries = fgq.Result,
SuccessGeneralQueries = sgq.Result
};
}
catch (Exception)
{
localToken.Cancel();
throw;
}
finally
{
runningTasks.SafeWaitAllToEnd();
runningTasks.DisposeTasks();
}

return null;
}

}
Expand Down
27 changes: 27 additions & 0 deletions Libs/Exceptions/ScanCrawlException.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace CxAnalytix.Exceptions
{
public class ScanCrawlException : Exception
{
private static String MakeMessage(String scanId, String projectName, String teamName) =>
$"Exception caught processing scan [{scanId}] in [{projectName}]" +
(!String.IsNullOrEmpty(teamName) ? $" assigned to team(s) [{teamName}]" : "");


public ScanCrawlException (String scanId, String projectName, String teamName)
: base(ScanCrawlException.MakeMessage(scanId, projectName, teamName) )
{

}
public ScanCrawlException(String scanId, String projectName, String teamName, Exception ex)
: base(ScanCrawlException.MakeMessage(scanId, projectName, teamName), ex)
{

}
}
}
12 changes: 8 additions & 4 deletions Libs/Executive/ExecuteLoop.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ namespace CxAnalytix.Executive
{
public class ExecuteLoop : ExecuteOnce
{
private static readonly ILog appLog = LogManager.GetLogger(typeof(ExecuteLoop));
private static readonly ILog _log = LogManager.GetLogger(typeof(ExecuteLoop));

public static new void Execute(CancellationTokenSource t)
{
Expand All @@ -35,19 +35,23 @@ public static new void Execute(CancellationTokenSource t)
}
catch (Exception ex)
{
appLog.Error("Vulnerability data transformation aborted due to unhandled exception.", ex);
_log.Error("Vulnerability data transformation aborted due to unhandled exception.", ex);
}


GC.Collect();

Task.Delay(Service.ProcessPeriodMinutes * 60 * 1000, t.Token).Wait();
using (var delay = Task.Delay(Service.ProcessPeriodMinutes * 60 * 1000, t.Token))
delay.Wait(t.Token);

} while (!t.Token.IsCancellationRequested);

_log.Info("Execution complete, ending.");

}
private static void Fatal(Exception ex, CancellationTokenSource ct)
{
appLog.Error("Fatal exception caught, program ending.", ex);
_log.Error("Fatal exception caught, program ending.", ex);
ct.Cancel();
Process.GetCurrentProcess().Kill(true);

Expand Down
Loading

0 comments on commit 37e85fd

Please sign in to comment.