cxone crawl stability

checkmarx-ts · Jul 18, 2023 · 0178d82 · 0178d82
1 parent bf4afe3
commit 0178d82
Show file tree

Hide file tree

Showing 5 changed files with 320 additions and 114 deletions.
diff --git a/Libs/Exceptions/ScanCrawlException.cs b/Libs/Exceptions/ScanCrawlException.cs
@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CxAnalytix.Exceptions
+{
+    public class ScanCrawlException : Exception
+    {
+        private static String MakeMessage(String scanId, String projectName, String teamName) =>
+            $"Exception caught processing scan [{scanId}] in [{projectName}]" +
+                                        (!String.IsNullOrEmpty(teamName) ? $" assigned to team(s) [{teamName}]" : "");
+
+
+        public ScanCrawlException (String scanId, String projectName, String teamName) 
+            : base(ScanCrawlException.MakeMessage(scanId, projectName, teamName) )
+        {
+
+        }
+        public ScanCrawlException(String scanId, String projectName, String teamName, Exception ex) 
+            : base(ScanCrawlException.MakeMessage(scanId, projectName, teamName), ex)
+        {
+
+        }
+    }
+}
diff --git a/Libs/Executive/ExecuteLoop.cs b/Libs/Executive/ExecuteLoop.cs
@@ -11,7 +11,7 @@ namespace CxAnalytix.Executive
 {
     public class ExecuteLoop : ExecuteOnce
     {
-        private static readonly ILog appLog = LogManager.GetLogger(typeof(ExecuteLoop));
+        private static readonly ILog _log = LogManager.GetLogger(typeof(ExecuteLoop));
 
         public static new void Execute(CancellationTokenSource t)
         {
@@ -35,19 +35,23 @@ public static new void Execute(CancellationTokenSource t)
                 }
                 catch (Exception ex)
                 {
-                    appLog.Error("Vulnerability data transformation aborted due to unhandled exception.", ex);
+                    _log.Error("Vulnerability data transformation aborted due to unhandled exception.", ex);
                 }
 
 
                 GC.Collect();
 
-                Task.Delay(Service.ProcessPeriodMinutes * 60 * 1000, t.Token).Wait();
+                using (var delay = Task.Delay(Service.ProcessPeriodMinutes * 60 * 1000, t.Token))
+                    delay.Wait(t.Token);
+
             } while (!t.Token.IsCancellationRequested);
 
+            _log.Info("Execution complete, ending.");
+
         }
         private static void Fatal(Exception ex, CancellationTokenSource ct)
         {
-            appLog.Error("Fatal exception caught, program ending.", ex);
+            _log.Error("Fatal exception caught, program ending.", ex);
             ct.Cancel();
             Process.GetCurrentProcess().Kill(true);
 

diff --git a/Libs/Executive/ExecuteOnce.cs b/Libs/Executive/ExecuteOnce.cs
@@ -64,6 +64,16 @@ private static IEnumerable<ITransformer> LoadTransformers(ILifetimeScope lifeSco
             return retVal;
         }
 
+        private static void LogExceptionAsError(int counter, Exception ex)
+        {
+            if (ex == null)
+                return;
+
+            _log.Error($"Exception Level {counter}", ex);
+
+            LogExceptionAsError(counter++, ex.InnerException);
+        }
+
         public static void Execute(CancellationTokenSource? t = null)
         {
             using (var scope = _xformersContainer.BeginLifetimeScope())
@@ -93,6 +103,22 @@ public static void Execute(CancellationTokenSource? t = null)
                     {
                         xformer.DoTransform(t.Token);
                     }
+                    catch (AggregateException aex)
+                    {
+                        _log.Error($"Unhandled exception when executing transformer module: {xformer.DisplayName}", aex);
+                        int counter = 0;
+
+
+                        foreach(var ex in aex.InnerExceptions)
+                        {
+                            _log.Error($"-- BEGIN AGGREGATE EXCEPTION {++counter} --");
+
+                            LogExceptionAsError(0, ex);
+
+                            _log.Error($"-- END AGGREGATE EXCEPTION {counter} --");
+                        }
+
+                    }
                     catch (Exception ex)
                     {
                         _log.Error($"Unhandled exception when executing transformer module: {xformer.DisplayName}", ex);

diff --git a/Libs/Extensions/Threading.cs b/Libs/Extensions/Threading.cs
@@ -11,16 +11,78 @@ public static class Threading
     {
         private static readonly int WAIT_MAX = 120000;
 
-        public static Task<T> DisposeTask<T>(this Task<T> task)
+        private static void doDispose(Task task)
         {
             if (task != null)
             {
                 task.Wait(WAIT_MAX);
                 if (task.IsCompleted)
                     task.Dispose();
             }
+        }
+
+        public static Task<T> DisposeTask<T>(this Task<T> task)
+        {
+            doDispose(task);
+            return null;
+        }
 
+        public static Task DisposeTask(this Task task)
+        {
+            doDispose(task);
             return null;
         }
+        public static void DisposeTasks(this IEnumerable<Task> tasks)
+        {
+            foreach(var task in tasks)
+                doDispose(task);
+        }
+
+        public static void SafeWaitToEnd(this Task task)
+        {
+            if (task == null)
+                return;
+
+            try
+            {
+                if (!task.IsCompleted)
+                    task.Wait();
+            }
+            // Eat the exceptions so it ensures all the tasks end.
+            catch (AggregateException)
+            {
+            }
+            catch (TaskCanceledException)
+            {
+            }
+        }
+
+
+        public static void SafeWaitAllToEnd(this IEnumerable<Task> tasks)
+        {
+            if (tasks == null)
+                return;
+
+            foreach (var task in tasks)
+            {
+                if (task == null)
+                    continue;
+
+                try
+                {
+                    if (!task.IsCompleted)
+                        task.Wait();
+                }
+                // Eat the exceptions so it ensures all the tasks end.
+                catch (AggregateException)
+                {
+                }
+                catch (TaskCanceledException)
+                {
+                }
+            }
+
+        }
+
     }
 }