From 9ecb3cfbac6157eaa6a57bda6244036d9d3be0a7 Mon Sep 17 00:00:00 2001 From: Claudia Murialdo Date: Thu, 31 Mar 2022 12:43:17 -0300 Subject: [PATCH 1/4] Re-implement PR #446 with a new solution which takes into account only the threads started by a Submit call type. It does not wait for other threads to finish (like the ones started by aspnet thread pool). --- .../GxClasses/Core/GXApplication.cs | 2 +- .../GxClasses/Core/GXUtilsCommon.cs | 24 ++++++++++++++++++- .../GxClasses/Model/GXWebProcedure.cs | 6 +++++ .../dotnetframework/GxClasses/Model/gxproc.cs | 7 ++++++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs index 4ca050297..501c98c0b 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs @@ -3414,7 +3414,7 @@ public IGxSession GetSession() } internal bool IsStandalone => this._session is GxSession || this._isSumbited || this.HttpContext == null; - + internal bool IsSubmited => this._isSumbited; internal void SetSession(IGxSession value) { if (value != null) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index f27795d21..990ce448b 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -55,7 +55,6 @@ public class GxDefaultProps public static string WORKSTATION = "GX_WRKST"; } - public class ThreadSafeRandom { private static RNGCryptoServiceProvider random = new RNGCryptoServiceProvider(); @@ -66,6 +65,8 @@ public static double NextDouble() byte[] bytes = new Byte[8]; random.GetBytes(bytes); + + ulong ul = BitConverter.ToUInt64(bytes, 0) / (1 << 11); double d = ul / (double)(1UL << 53); return d; @@ -5721,5 +5722,26 @@ public static string EncodeNonAsciiCharacters(string value) return sb.ToString(); } } + internal class ThreadUtil + { + private static List events = new List(); + + internal static void Submit(WaitCallback callbak, object state) + { + var resetEvent = new ManualResetEvent(false); + ThreadPool.QueueUserWorkItem( + arg => + { + callbak(state); + resetEvent.Set(); + }); + events.Add(resetEvent); + } + internal static void WaitForEnd() + { + if (events.Count > 0) + WaitHandle.WaitAll(events.ToArray()); + } + } } \ No newline at end of file diff --git a/dotnet/src/dotnetframework/GxClasses/Model/GXWebProcedure.cs b/dotnet/src/dotnetframework/GxClasses/Model/GXWebProcedure.cs index da4ceea02..0b7352fde 100644 --- a/dotnet/src/dotnetframework/GxClasses/Model/GXWebProcedure.cs +++ b/dotnet/src/dotnetframework/GxClasses/Model/GXWebProcedure.cs @@ -7,6 +7,7 @@ namespace GeneXus.Procedure using GeneXus.Http; using System.Threading; using GeneXus.Mime; + using GeneXus.Utils; public class GXWebProcedure : GXHttpHandler { @@ -191,5 +192,10 @@ protected static WaitCallback PropagateCulture(WaitCallback action) { return GXProcedure.PropagateCulture(action); } + protected void Submit(Action executeMethod, object state) + { + ThreadUtil.Submit(PropagateCulture(new WaitCallback(executeMethod)), state); + } + } } \ No newline at end of file diff --git a/dotnet/src/dotnetframework/GxClasses/Model/gxproc.cs b/dotnet/src/dotnetframework/GxClasses/Model/gxproc.cs index a44702de1..adf4adb96 100644 --- a/dotnet/src/dotnetframework/GxClasses/Model/gxproc.cs +++ b/dotnet/src/dotnetframework/GxClasses/Model/gxproc.cs @@ -65,6 +65,10 @@ public bool DisconnectAtCleanup get{ return disconnectUserAtCleanup;} set{ disconnectUserAtCleanup=value;} } + protected void Submit(Action executeMethod, object state) + { + ThreadUtil.Submit(PropagateCulture(new WaitCallback(executeMethod)), state); + } public static WaitCallback PropagateCulture(WaitCallback action) { var currentCulture = Thread.CurrentThread.CurrentCulture; @@ -87,6 +91,9 @@ protected void exitApplication() } private void exitApplication(bool flushBatchCursor) { + if (IsMain && !(context as GxContext).IsSubmited) + ThreadUtil.WaitForEnd(); + if (flushBatchCursor) { foreach (IGxDataStore ds in context.DataStores) From 3bada808049151e7aa4f8d9c15a0b36de3e208ff Mon Sep 17 00:00:00 2001 From: Claudia Murialdo Date: Thu, 31 Mar 2022 17:27:40 -0300 Subject: [PATCH 2/4] Clear events after WaitAll ending. --- dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 990ce448b..3d9b7a95a 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -5728,7 +5728,7 @@ internal class ThreadUtil internal static void Submit(WaitCallback callbak, object state) { - var resetEvent = new ManualResetEvent(false); + ManualResetEvent resetEvent = new ManualResetEvent(false); ThreadPool.QueueUserWorkItem( arg => { @@ -5740,7 +5740,10 @@ internal static void Submit(WaitCallback callbak, object state) internal static void WaitForEnd() { if (events.Count > 0) + { WaitHandle.WaitAll(events.ToArray()); + events.Clear(); + } } } From d21ad066491f5f1db0d55a9cb5e0c4f92ccfaa35 Mon Sep 17 00:00:00 2001 From: Claudia Murialdo Date: Thu, 31 Mar 2022 19:24:14 -0300 Subject: [PATCH 3/4] Add log message to submit waiting routine. --- .../GxClasses/Core/GXUtilsCommon.cs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 3d9b7a95a..9171496ac 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -5724,6 +5724,7 @@ public static string EncodeNonAsciiCharacters(string value) } internal class ThreadUtil { + static readonly ILog log = log4net.LogManager.GetLogger(typeof(ThreadUtil)); private static List events = new List(); internal static void Submit(WaitCallback callbak, object state) @@ -5739,12 +5740,27 @@ internal static void Submit(WaitCallback callbak, object state) } internal static void WaitForEnd() { + int remainingSubmits = 0; if (events.Count > 0) { - WaitHandle.WaitAll(events.ToArray()); + foreach (ManualResetEvent e in events) + { + if (IsPending(e)) + remainingSubmits++; + } + if (remainingSubmits>0) + { + GXLogging.Debug(log, "Waiting for " + remainingSubmits + " submitted procs to end..."); + WaitHandle.WaitAll(events.ToArray()); + } events.Clear(); } } + static bool IsPending(ManualResetEvent eventState) + { + return !eventState.WaitOne(0); + + } } } \ No newline at end of file From c55408d5338f20c6768fca8c3c1d322ea1c1dee5 Mon Sep 17 00:00:00 2001 From: Claudia Murialdo Date: Tue, 26 Apr 2022 09:20:00 -0300 Subject: [PATCH 4/4] Add unit test for submit function. Take into account limitation of WaitHandle.WaitAll and 64 threads. --- .../GxClasses/Core/GXApplication.cs | 2 + .../GxClasses/Core/GXUtilsCommon.cs | 72 ++++++++++----- .../DotNetCoreUnitTest.csproj | 2 +- .../DotNetCoreUnitTest/Submit/SubmitTest.cs | 88 +++++++++++++++++++ 4 files changed, 139 insertions(+), 25 deletions(-) create mode 100644 dotnet/test/DotNetCoreUnitTest/Submit/SubmitTest.cs diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs index 501c98c0b..219769018 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXApplication.cs @@ -2102,6 +2102,8 @@ public virtual short GetHttpSecure() { try { + if (HttpContext==null) + return 0; if (_HttpContext.Request.GetIsSecureFrontEnd()) { GXLogging.Debug(log, "Front-End-Https header activated"); diff --git a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs index 9171496ac..9fa9b0d67 100644 --- a/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs +++ b/dotnet/src/dotnetframework/GxClasses/Core/GXUtilsCommon.cs @@ -43,6 +43,7 @@ using GeneXus.Services; using GeneXus.Http; using System.Security; +using System.Threading.Tasks; namespace GeneXus.Utils { @@ -5725,41 +5726,64 @@ public static string EncodeNonAsciiCharacters(string value) internal class ThreadUtil { static readonly ILog log = log4net.LogManager.GetLogger(typeof(ThreadUtil)); - private static List events = new List(); - + private static ConcurrentDictionary events = new ConcurrentDictionary(); + const int MAX_WAIT_HANDLES = 64; internal static void Submit(WaitCallback callbak, object state) { - ManualResetEvent resetEvent = new ManualResetEvent(false); - ThreadPool.QueueUserWorkItem( - arg => - { - callbak(state); - resetEvent.Set(); - }); - events.Add(resetEvent); + try + { + ManualResetEvent resetEvent = new ManualResetEvent(false); + Guid eventGuid = Guid.NewGuid(); + ThreadPool.QueueUserWorkItem( + arg => + { + callbak(state); + resetEvent.Set(); + events.TryRemove(eventGuid, out ManualResetEvent _); + + }); + events[eventGuid]= resetEvent; + } + catch (Exception ex) + { + GXLogging.Error(log, $"Submit error", ex); + } } internal static void WaitForEnd() { - int remainingSubmits = 0; - if (events.Count > 0) + int remainingSubmits = events.Count; + if (remainingSubmits > 0) { - foreach (ManualResetEvent e in events) + GXLogging.Debug(log, "Waiting for " + remainingSubmits + " submitted procs to end..."); + WaitForAll(); + events.Clear(); + } + } + public static void WaitForAll() + { + try + { + List evtList =new List (); + foreach (ManualResetEvent evt in events.Values) { - if (IsPending(e)) - remainingSubmits++; + evtList.Add(evt); + //Avoid WaitHandle.WaitAll limitation. It can only handle 64 waithandles + if (evtList.Count == MAX_WAIT_HANDLES) + { + WaitHandle.WaitAll(evtList.ToArray()); + evtList.Clear(); + } } - if (remainingSubmits>0) + if (evtList.Count>0) { - GXLogging.Debug(log, "Waiting for " + remainingSubmits + " submitted procs to end..."); - WaitHandle.WaitAll(events.ToArray()); + WaitHandle.WaitAll(evtList.ToArray()); + evtList.Clear(); } - events.Clear(); } - } - static bool IsPending(ManualResetEvent eventState) - { - return !eventState.WaitOne(0); - + catch (Exception ex) + { + GXLogging.Error(log, $"WaitForAll pending threads error", ex); + } } } diff --git a/dotnet/test/DotNetCoreUnitTest/DotNetCoreUnitTest.csproj b/dotnet/test/DotNetCoreUnitTest/DotNetCoreUnitTest.csproj index fb6bc20af..0a55d803a 100644 --- a/dotnet/test/DotNetCoreUnitTest/DotNetCoreUnitTest.csproj +++ b/dotnet/test/DotNetCoreUnitTest/DotNetCoreUnitTest.csproj @@ -1,4 +1,4 @@ - + net6.0 CS8032;1701;1702;NU1701 diff --git a/dotnet/test/DotNetCoreUnitTest/Submit/SubmitTest.cs b/dotnet/test/DotNetCoreUnitTest/Submit/SubmitTest.cs new file mode 100644 index 000000000..546dc43d7 --- /dev/null +++ b/dotnet/test/DotNetCoreUnitTest/Submit/SubmitTest.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using GeneXus.Application; +using GeneXus.Procedure; +using GeneXus.Utils; +using Lucene.Net.Support; +using Xunit; +using System.Collections.Concurrent; + + +namespace UnitTesting +{ + public class SubmitTest + { + internal static ConcurrentDictionary numbers = new ConcurrentDictionary(); + [Fact] + public void SubmitAll() + { + MainProc proc = new MainProc(); + proc.execute(); + } + } + public class MainProc : GXProcedure + { + const int THREAD_NUMBER = 2000; + public MainProc() + { + context = new GxContext(); + IsMain = true; + } + public void execute() + { + initialize(); + executePrivate(); + } + + void executePrivate() + { + for (int i = 0; i < THREAD_NUMBER; i++) + { + SubmitProc proc = new SubmitProc(); + proc.executeSubmit(); + } + cleanup(); + Assert.Equal(THREAD_NUMBER, SubmitTest.numbers.Count); + } + public override void cleanup() + { + ExitApp(); + } + + public override void initialize() + { + + } + } + public class SubmitProc : GXProcedure + { + public SubmitProc() + { + context = new GxContext(); + } + public void executeSubmit() + { + SubmitProc submitProc; + submitProc = new SubmitProc(); + submitProc.context.SetSubmitInitialConfig(context); + submitProc.initialize(); + Submit(executePrivateCatch, submitProc); + } + void executePrivateCatch(object stateInfo) + { + ((SubmitProc)stateInfo).executePrivate(); + } + void executePrivate() + { + int millisecondsToWait = (int)ThreadSafeRandom.NextDouble()*10; + Thread.Sleep(millisecondsToWait); + SubmitTest.numbers[Guid.NewGuid()]=Thread.CurrentThread.ManagedThreadId; + } + + public override void initialize() + { + + } + } +}