From ad09e392a003b6d4dd95dfcba7416feb7f56d9d9 Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Mon, 27 May 2019 11:26:44 -0700 Subject: [PATCH 01/14] HW-53073: Allow Thread Interruption for WriteLockManager --- .gitignore | 2 ++ .../persistence/internal/helper/WriteLockManager.java | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3a367cf1038..9bc95214ebb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.iml +.idea/* .DS_Store *.class *test.jar diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index 7cc2d566d94..789110bf4f9 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.logging.Logger; import org.eclipse.persistence.descriptors.ClassDescriptor; import org.eclipse.persistence.descriptors.FetchGroupManager; @@ -60,6 +61,7 @@ * @since 10.0.3 */ public class WriteLockManager { + private static Logger logger = Logger.getLogger(WriteLockManager.class.getSimpleName()); // this will allow us to prevent a readlock thread from looping forever. public static final int MAXTRIES = 10000; @@ -72,6 +74,8 @@ public class WriteLockManager { public WriteLockManager() { this.prevailingQueue = new ExposedNodeLinkedList(); + logger.warning("THIS IS A CUSTOM FORK FOR ECLIPSELINK v2.7.3"); + } /** @@ -98,7 +102,9 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto toWaitOn.wait();// wait for lock on object to be released } } catch (InterruptedException ex) { - // Ignore exception thread should continue. + //https://jira.site1.hyperwallet.local/browse/HW-53073 + //Custom change to allow thread interruptions for bad threads stuck in org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone pattern + throw org.eclipse.persistence.exceptions.ConcurrencyException.waitWasInterrupted(ex.getMessage()); } } Object waitObject = toWaitOn.getObject(); From e9e72aed91e587b8236b26e268372900063928e7 Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Mon, 3 Jun 2019 16:05:18 -0700 Subject: [PATCH 02/14] more logging. --- .../persistence/internal/helper/WriteLockManager.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index 789110bf4f9..dd59fe5bb01 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -22,10 +22,13 @@ // - 526957 : Split the logging and trace messages package org.eclipse.persistence.internal.helper; +import java.time.Duration; +import java.time.Instant; import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.logging.Level; import java.util.logging.Logger; import org.eclipse.persistence.descriptors.ClassDescriptor; @@ -88,6 +91,7 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto IdentityHashMap lockedObjects = new IdentityHashMap(); IdentityHashMap refreshedObjects = new IdentityHashMap(); try { + Instant startTime = Instant.now(); // if the descriptor has indirection for all mappings then wait as there will be no deadlock risks CacheKey toWaitOn = acquireLockAndRelatedLocks(objectForClone, lockedObjects, refreshedObjects, cacheKey, descriptor, cloningSession); int tries = 0; @@ -102,9 +106,12 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto toWaitOn.wait();// wait for lock on object to be released } } catch (InterruptedException ex) { + Instant interruptionTime = Instant.now(); + logger.log(Level.WARNING, "This is a custom eclipselink change to allow interrupts after tries: {0}, time taken: {1}", new Object[]{tries, + Duration.between(startTime, interruptionTime)}); //https://jira.site1.hyperwallet.local/browse/HW-53073 //Custom change to allow thread interruptions for bad threads stuck in org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone pattern - throw org.eclipse.persistence.exceptions.ConcurrencyException.waitWasInterrupted(ex.getMessage()); + throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); } } Object waitObject = toWaitOn.getObject(); From fb013ff2a3630486b6a15a3a8c0a2d3106f38c5f Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Mon, 3 Jun 2019 16:16:32 -0700 Subject: [PATCH 03/14] more logging. --- .../eclipse/persistence/internal/helper/WriteLockManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index dd59fe5bb01..12cbde8e3a7 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -77,7 +77,6 @@ public class WriteLockManager { public WriteLockManager() { this.prevailingQueue = new ExposedNodeLinkedList(); - logger.warning("THIS IS A CUSTOM FORK FOR ECLIPSELINK v2.7.3"); } From 2d68361829b24036f0ec51c0ebddbb571a23a3c8 Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Mon, 3 Jun 2019 16:18:02 -0700 Subject: [PATCH 04/14] more logging. --- .../eclipse/persistence/internal/helper/WriteLockManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index 12cbde8e3a7..cf0b46bdaa2 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -77,7 +77,6 @@ public class WriteLockManager { public WriteLockManager() { this.prevailingQueue = new ExposedNodeLinkedList(); - } /** From ef3768313db9f780e4b339cb2f0ea9a41b4c8f91 Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Tue, 4 Jun 2019 12:07:03 -0700 Subject: [PATCH 05/14] wait 1 min before interrupt. --- .../internal/helper/WriteLockManager.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index cf0b46bdaa2..78f212735a9 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -70,6 +70,7 @@ public class WriteLockManager { public static final int MAXTRIES = 10000; public static final int MAX_WAIT = 600000; //10 mins + private static final int ONE_MINUTE_WAIT = 60_000; //1 minute /* This attribute stores the list of threads that have had a problem acquiring locks */ /* the first element in this list will be the prevailing thread */ @@ -89,7 +90,7 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto IdentityHashMap lockedObjects = new IdentityHashMap(); IdentityHashMap refreshedObjects = new IdentityHashMap(); try { - Instant startTime = Instant.now(); + long startTimeInMillis = System.currentTimeMillis(); // if the descriptor has indirection for all mappings then wait as there will be no deadlock risks CacheKey toWaitOn = acquireLockAndRelatedLocks(objectForClone, lockedObjects, refreshedObjects, cacheKey, descriptor, cloningSession); int tries = 0; @@ -104,12 +105,18 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto toWaitOn.wait();// wait for lock on object to be released } } catch (InterruptedException ex) { - Instant interruptionTime = Instant.now(); - logger.log(Level.WARNING, "This is a custom eclipselink change to allow interrupts after tries: {0}, time taken: {1}", new Object[]{tries, - Duration.between(startTime, interruptionTime)}); //https://jira.site1.hyperwallet.local/browse/HW-53073 //Custom change to allow thread interruptions for bad threads stuck in org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone pattern - throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); + Long secondsElapsed = (System.currentTimeMillis() - startTimeInMillis) / 1000; + + if (secondsElapsed >= ONE_MINUTE_WAIT) { + logger.log(Level.SEVERE, "Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}", new Object[]{tries, + secondsElapsed}); + throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); + } else { + logger.log(Level.WARNING, "This is a custom eclipselink change to allow interrupts, it will not interrupt till 1 minute. Attempts: {0}, Time elapsed in seconds: {1}", new Object[]{tries, + secondsElapsed}); + } } } Object waitObject = toWaitOn.getObject(); From fdb49706bd16350cf8a56624e28569a4feb5affa Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Tue, 4 Jun 2019 12:10:12 -0700 Subject: [PATCH 06/14] wait 1 min before interrupt. --- .../eclipse/persistence/internal/helper/WriteLockManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index 78f212735a9..ebe262b9519 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -70,7 +70,7 @@ public class WriteLockManager { public static final int MAXTRIES = 10000; public static final int MAX_WAIT = 600000; //10 mins - private static final int ONE_MINUTE_WAIT = 60_000; //1 minute + private static final int MAX_SECS_WAIT = 60; //1 minute /* This attribute stores the list of threads that have had a problem acquiring locks */ /* the first element in this list will be the prevailing thread */ @@ -109,7 +109,7 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto //Custom change to allow thread interruptions for bad threads stuck in org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone pattern Long secondsElapsed = (System.currentTimeMillis() - startTimeInMillis) / 1000; - if (secondsElapsed >= ONE_MINUTE_WAIT) { + if (secondsElapsed >= MAX_SECS_WAIT) { logger.log(Level.SEVERE, "Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}", new Object[]{tries, secondsElapsed}); throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); From c4663eb62b6ad0089bf67c2c6a934eb80abd76aa Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Mon, 17 Jun 2019 22:49:09 -0700 Subject: [PATCH 07/14] writelock stuff changes. --- .../internal/helper/WriteLockManager.java | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index ebe262b9519..b2395673c10 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -22,31 +22,26 @@ // - 526957 : Split the logging and trace messages package org.eclipse.persistence.internal.helper; -import java.time.Duration; -import java.time.Instant; -import java.util.IdentityHashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - import org.eclipse.persistence.descriptors.ClassDescriptor; import org.eclipse.persistence.descriptors.FetchGroupManager; import org.eclipse.persistence.exceptions.ConcurrencyException; import org.eclipse.persistence.internal.helper.linkedlist.ExposedNodeLinkedList; import org.eclipse.persistence.internal.identitymaps.CacheKey; -import org.eclipse.persistence.internal.localization.LoggingLocalization; import org.eclipse.persistence.internal.localization.TraceLocalization; import org.eclipse.persistence.internal.queries.ContainerPolicy; -import org.eclipse.persistence.internal.sessions.AbstractSession; -import org.eclipse.persistence.internal.sessions.MergeManager; -import org.eclipse.persistence.internal.sessions.ObjectChangeSet; -import org.eclipse.persistence.internal.sessions.UnitOfWorkChangeSet; -import org.eclipse.persistence.internal.sessions.UnitOfWorkImpl; +import org.eclipse.persistence.internal.sessions.*; import org.eclipse.persistence.logging.SessionLog; import org.eclipse.persistence.mappings.DatabaseMapping; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + /** * INTERNAL: *

@@ -70,7 +65,11 @@ public class WriteLockManager { public static final int MAXTRIES = 10000; public static final int MAX_WAIT = 600000; //10 mins - private static final int MAX_SECS_WAIT = 60; //1 minute + private static final int SIXTY_SECONDS = 60; //1 minute + private static final int SEVENTY_SECONDS = 70; //Once we reached 70s, we can assume we are in a bad state, so allow interruption for everything. + //Initialize with 0 when we first find a lock. + //is volatile enough for this case? How do we guarantee Thread-Safe access? Do we want to add more complexity with synchronized + private static volatile LocalDateTime honourInterruptsWithin70secondsOfThisTime = LocalDateTime.now().minusYears(1); /* This attribute stores the list of threads that have had a problem acquiring locks */ /* the first element in this list will be the prevailing thread */ @@ -90,7 +89,7 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto IdentityHashMap lockedObjects = new IdentityHashMap(); IdentityHashMap refreshedObjects = new IdentityHashMap(); try { - long startTimeInMillis = System.currentTimeMillis(); + LocalDateTime localTime = null; // if the descriptor has indirection for all mappings then wait as there will be no deadlock risks CacheKey toWaitOn = acquireLockAndRelatedLocks(objectForClone, lockedObjects, refreshedObjects, cacheKey, descriptor, cloningSession); int tries = 0; @@ -107,15 +106,30 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto } catch (InterruptedException ex) { //https://jira.site1.hyperwallet.local/browse/HW-53073 //Custom change to allow thread interruptions for bad threads stuck in org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone pattern - Long secondsElapsed = (System.currentTimeMillis() - startTimeInMillis) / 1000; - - if (secondsElapsed >= MAX_SECS_WAIT) { - logger.log(Level.SEVERE, "Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}", new Object[]{tries, - secondsElapsed}); + long secondsPassed = ChronoUnit.SECONDS.between(honourInterruptsWithin70secondsOfThisTime, LocalDateTime.now()); + if (secondsPassed < SEVENTY_SECONDS) { + logger.log(Level.SEVERE, "Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Timer: {2}", + new Object[]{tries, + secondsPassed, honourInterruptsWithin70secondsOfThisTime}); throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); } else { - logger.log(Level.WARNING, "This is a custom eclipselink change to allow interrupts, it will not interrupt till 1 minute. Attempts: {0}, Time elapsed in seconds: {1}", new Object[]{tries, - secondsElapsed}); + if (localTime == null) { + localTime = LocalDateTime.now(); + } + long secondsPassedLocally = ChronoUnit.SECONDS.between(localTime, LocalDateTime.now()); + if (secondsPassedLocally < SIXTY_SECONDS) { + logger.log(Level.WARNING, + "This is a custom eclipselink change to allow interrupts, it will not interrupt till 1 minute. Attempts: " + + "{0}, Time elapsed in seconds: {1}", + new Object[]{tries, + secondsPassedLocally}); + } else { + honourInterruptsWithin70secondsOfThisTime = LocalDateTime.now(); + logger.log(Level.SEVERE, "Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Timer: {2}", + new Object[]{tries, + secondsPassed, honourInterruptsWithin70secondsOfThisTime}); + throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); + } } } } From 965591de3c8ddb36c1b4720152dd77138eff644f Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Wed, 19 Jun 2019 16:31:08 -0700 Subject: [PATCH 08/14] log changes --- .../persistence/internal/helper/WriteLockManager.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index b2395673c10..27824144848 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -108,7 +108,8 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto //Custom change to allow thread interruptions for bad threads stuck in org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone pattern long secondsPassed = ChronoUnit.SECONDS.between(honourInterruptsWithin70secondsOfThisTime, LocalDateTime.now()); if (secondsPassed < SEVENTY_SECONDS) { - logger.log(Level.SEVERE, "Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Timer: {2}", + logger.log(Level.SEVERE, + "Static Timer: Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Timer: {2}", new Object[]{tries, secondsPassed, honourInterruptsWithin70secondsOfThisTime}); throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); @@ -125,7 +126,8 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto secondsPassedLocally}); } else { honourInterruptsWithin70secondsOfThisTime = LocalDateTime.now(); - logger.log(Level.SEVERE, "Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Timer: {2}", + logger.log(Level.SEVERE, + "Local Timer: Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Timer: {2}", new Object[]{tries, secondsPassed, honourInterruptsWithin70secondsOfThisTime}); throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); From 35bff2a75d7453a21f57bc8e0ec74b14f55041a8 Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Mon, 24 Jun 2019 11:54:29 -0700 Subject: [PATCH 09/14] log changes --- .../persistence/internal/helper/WriteLockManager.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index 27824144848..2566725aa77 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -121,15 +121,14 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto if (secondsPassedLocally < SIXTY_SECONDS) { logger.log(Level.WARNING, "This is a custom eclipselink change to allow interrupts, it will not interrupt till 1 minute. Attempts: " - + "{0}, Time elapsed in seconds: {1}", - new Object[]{tries, - secondsPassedLocally}); + + "{0}, Time elapsed in seconds: {1}, Local Timer: {2}, Static Timer: {3}", + new Object[]{tries, secondsPassedLocally, localTime, honourInterruptsWithin70secondsOfThisTime}); } else { honourInterruptsWithin70secondsOfThisTime = LocalDateTime.now(); logger.log(Level.SEVERE, - "Local Timer: Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Timer: {2}", - new Object[]{tries, - secondsPassed, honourInterruptsWithin70secondsOfThisTime}); + "Local Timer: Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Local Timer: " + + "{2}, Static Timer: {3}", + new Object[]{tries, secondsPassed, localTime, honourInterruptsWithin70secondsOfThisTime}); throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); } } From 4db2cb356919d3fbf997e5eb63fc08c5718342f4 Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Thu, 4 Jul 2019 11:47:29 -0700 Subject: [PATCH 10/14] bump 70 to 130 --- .../eclipse/persistence/internal/helper/WriteLockManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index 2566725aa77..fb66f754ea8 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -66,7 +66,7 @@ public class WriteLockManager { public static final int MAX_WAIT = 600000; //10 mins private static final int SIXTY_SECONDS = 60; //1 minute - private static final int SEVENTY_SECONDS = 70; //Once we reached 70s, we can assume we are in a bad state, so allow interruption for everything. + private static final int SEVENTY_SECONDS = 130; //Once we reached 130s, we can assume we are in a bad state, so allow interruption for everything. //Initialize with 0 when we first find a lock. //is volatile enough for this case? How do we guarantee Thread-Safe access? Do we want to add more complexity with synchronized private static volatile LocalDateTime honourInterruptsWithin70secondsOfThisTime = LocalDateTime.now().minusYears(1); @@ -124,11 +124,11 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto + "{0}, Time elapsed in seconds: {1}, Local Timer: {2}, Static Timer: {3}", new Object[]{tries, secondsPassedLocally, localTime, honourInterruptsWithin70secondsOfThisTime}); } else { - honourInterruptsWithin70secondsOfThisTime = LocalDateTime.now(); logger.log(Level.SEVERE, "Local Timer: Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Local Timer: " + "{2}, Static Timer: {3}", new Object[]{tries, secondsPassed, localTime, honourInterruptsWithin70secondsOfThisTime}); + honourInterruptsWithin70secondsOfThisTime = LocalDateTime.now(); throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); } } From 5cd96c3090fe970eb64dce5d706017a3c46a7527 Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Thu, 4 Jul 2019 12:28:58 -0700 Subject: [PATCH 11/14] renames. --- .../persistence/internal/helper/WriteLockManager.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index fb66f754ea8..c17426f76aa 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -65,8 +65,11 @@ public class WriteLockManager { public static final int MAXTRIES = 10000; public static final int MAX_WAIT = 600000; //10 mins - private static final int SIXTY_SECONDS = 60; //1 minute - private static final int SEVENTY_SECONDS = 130; //Once we reached 130s, we can assume we are in a bad state, so allow interruption for everything. + private static final int SECONDS_IN_BAD_STATE = 60; //1 minute + //Once we reached SECONDS_BEFORE_HONOUR_INTERRUPT, we can assume the node is in a bad state for SECONDS_IN_BAD_STATE. During this time, we will + // honour all interrupts. + private static final int SECONDS_BEFORE_HONOUR_INTERRUPT = 130; + //Initialize with 0 when we first find a lock. //is volatile enough for this case? How do we guarantee Thread-Safe access? Do we want to add more complexity with synchronized private static volatile LocalDateTime honourInterruptsWithin70secondsOfThisTime = LocalDateTime.now().minusYears(1); @@ -107,7 +110,7 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto //https://jira.site1.hyperwallet.local/browse/HW-53073 //Custom change to allow thread interruptions for bad threads stuck in org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone pattern long secondsPassed = ChronoUnit.SECONDS.between(honourInterruptsWithin70secondsOfThisTime, LocalDateTime.now()); - if (secondsPassed < SEVENTY_SECONDS) { + if (secondsPassed < SECONDS_BEFORE_HONOUR_INTERRUPT) { logger.log(Level.SEVERE, "Static Timer: Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Timer: {2}", new Object[]{tries, @@ -118,7 +121,7 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto localTime = LocalDateTime.now(); } long secondsPassedLocally = ChronoUnit.SECONDS.between(localTime, LocalDateTime.now()); - if (secondsPassedLocally < SIXTY_SECONDS) { + if (secondsPassedLocally < SECONDS_IN_BAD_STATE) { logger.log(Level.WARNING, "This is a custom eclipselink change to allow interrupts, it will not interrupt till 1 minute. Attempts: " + "{0}, Time elapsed in seconds: {1}, Local Timer: {2}, Static Timer: {3}", From d22ff4a1288b5d82461bd0c875cef8da10c33014 Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Thu, 4 Jul 2019 12:32:07 -0700 Subject: [PATCH 12/14] renames. --- .../eclipse/persistence/internal/helper/WriteLockManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index c17426f76aa..6dd59e0f441 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -65,7 +65,7 @@ public class WriteLockManager { public static final int MAXTRIES = 10000; public static final int MAX_WAIT = 600000; //10 mins - private static final int SECONDS_IN_BAD_STATE = 60; //1 minute + private static final int SECONDS_IN_BAD_STATE = 60; //Once we reached SECONDS_BEFORE_HONOUR_INTERRUPT, we can assume the node is in a bad state for SECONDS_IN_BAD_STATE. During this time, we will // honour all interrupts. private static final int SECONDS_BEFORE_HONOUR_INTERRUPT = 130; From c6f50eff79ab6abefe75d48a8ec7b59ed05c572f Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Mon, 22 Jul 2019 15:22:27 -0700 Subject: [PATCH 13/14] new logic with flag based conditionals. --- .../internal/helper/WriteLockManager.java | 46 ++++++------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index 6dd59e0f441..9892565f55e 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -33,8 +33,6 @@ import org.eclipse.persistence.logging.SessionLog; import org.eclipse.persistence.mappings.DatabaseMapping; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; @@ -65,14 +63,15 @@ public class WriteLockManager { public static final int MAXTRIES = 10000; public static final int MAX_WAIT = 600000; //10 mins - private static final int SECONDS_IN_BAD_STATE = 60; - //Once we reached SECONDS_BEFORE_HONOUR_INTERRUPT, we can assume the node is in a bad state for SECONDS_IN_BAD_STATE. During this time, we will - // honour all interrupts. - private static final int SECONDS_BEFORE_HONOUR_INTERRUPT = 130; + private static volatile boolean interruptionEnabled; - //Initialize with 0 when we first find a lock. - //is volatile enough for this case? How do we guarantee Thread-Safe access? Do we want to add more complexity with synchronized - private static volatile LocalDateTime honourInterruptsWithin70secondsOfThisTime = LocalDateTime.now().minusYears(1); + public static boolean isInterruptionEnabled() { + return interruptionEnabled; + } + + public static void setInterruptionEnabled(boolean interruptionEnabled) { + WriteLockManager.interruptionEnabled = interruptionEnabled; + } /* This attribute stores the list of threads that have had a problem acquiring locks */ /* the first element in this list will be the prevailing thread */ @@ -92,7 +91,6 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto IdentityHashMap lockedObjects = new IdentityHashMap(); IdentityHashMap refreshedObjects = new IdentityHashMap(); try { - LocalDateTime localTime = null; // if the descriptor has indirection for all mappings then wait as there will be no deadlock risks CacheKey toWaitOn = acquireLockAndRelatedLocks(objectForClone, lockedObjects, refreshedObjects, cacheKey, descriptor, cloningSession); int tries = 0; @@ -109,32 +107,13 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto } catch (InterruptedException ex) { //https://jira.site1.hyperwallet.local/browse/HW-53073 //Custom change to allow thread interruptions for bad threads stuck in org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone pattern - long secondsPassed = ChronoUnit.SECONDS.between(honourInterruptsWithin70secondsOfThisTime, LocalDateTime.now()); - if (secondsPassed < SECONDS_BEFORE_HONOUR_INTERRUPT) { + if (interruptionEnabled) { logger.log(Level.SEVERE, - "Static Timer: Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Timer: {2}", - new Object[]{tries, - secondsPassed, honourInterruptsWithin70secondsOfThisTime}); + "EclipseLinkLockHandler has set interruption flag to TRUE. Allowing interrupts"); throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); - } else { - if (localTime == null) { - localTime = LocalDateTime.now(); - } - long secondsPassedLocally = ChronoUnit.SECONDS.between(localTime, LocalDateTime.now()); - if (secondsPassedLocally < SECONDS_IN_BAD_STATE) { - logger.log(Level.WARNING, - "This is a custom eclipselink change to allow interrupts, it will not interrupt till 1 minute. Attempts: " - + "{0}, Time elapsed in seconds: {1}, Local Timer: {2}, Static Timer: {3}", - new Object[]{tries, secondsPassedLocally, localTime, honourInterruptsWithin70secondsOfThisTime}); - } else { - logger.log(Level.SEVERE, - "Local Timer: Reached threshold to interrupt. Attempts: {0}, Time elapsed in seconds: {1}, Local Timer: " - + "{2}, Static Timer: {3}", - new Object[]{tries, secondsPassed, localTime, honourInterruptsWithin70secondsOfThisTime}); - honourInterruptsWithin70secondsOfThisTime = LocalDateTime.now(); - throw ConcurrencyException.waitWasInterrupted(ex.getMessage()); - } } + logger.log(Level.WARNING, + "EclipseLinkLockHandler has not set the interruption flag to TRUE. Will not interrupt"); } } Object waitObject = toWaitOn.getObject(); @@ -146,6 +125,7 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto toWaitOn = acquireLockAndRelatedLocks(objectForClone, lockedObjects, refreshedObjects, cacheKey, descriptor, cloningSession); if ((toWaitOn != null) && ((++tries) > MAXTRIES)) { // If we've tried too many times abort. + logger.log(Level.WARNING, "Interruption done by EclipseLink Library."); throw ConcurrencyException.maxTriesLockOnCloneExceded(objectForClone); } } From 5d35263813c924058530465d5212e5bffb1d023d Mon Sep 17 00:00:00 2001 From: Anshu Gupta Date: Mon, 22 Jul 2019 17:40:44 -0700 Subject: [PATCH 14/14] new logic with flag based conditionals. --- .../internal/helper/WriteLockManager.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java index 9892565f55e..b24acef7457 100644 --- a/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java +++ b/foundation/org.eclipse.persistence.core/src/org/eclipse/persistence/internal/helper/WriteLockManager.java @@ -40,6 +40,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import static java.lang.Boolean.parseBoolean; + /** * INTERNAL: *

@@ -57,21 +59,19 @@ * @since 10.0.3 */ public class WriteLockManager { - private static Logger logger = Logger.getLogger(WriteLockManager.class.getSimpleName()); + + private static final Logger logger = Logger.getLogger(WriteLockManager.class.getSimpleName()); // this will allow us to prevent a readlock thread from looping forever. public static final int MAXTRIES = 10000; public static final int MAX_WAIT = 600000; //10 mins - private static volatile boolean interruptionEnabled; - public static boolean isInterruptionEnabled() { - return interruptionEnabled; + private static boolean isInterruptionEnabled() { + String value = System.getProperty("interruptionEnabled"); + return parseBoolean(value); } - public static void setInterruptionEnabled(boolean interruptionEnabled) { - WriteLockManager.interruptionEnabled = interruptionEnabled; - } /* This attribute stores the list of threads that have had a problem acquiring locks */ /* the first element in this list will be the prevailing thread */ @@ -107,7 +107,7 @@ public Map acquireLocksForClone(Object objectForClone, ClassDescriptor descripto } catch (InterruptedException ex) { //https://jira.site1.hyperwallet.local/browse/HW-53073 //Custom change to allow thread interruptions for bad threads stuck in org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone pattern - if (interruptionEnabled) { + if (isInterruptionEnabled()) { logger.log(Level.SEVERE, "EclipseLinkLockHandler has set interruption flag to TRUE. Allowing interrupts"); throw ConcurrencyException.waitWasInterrupted(ex.getMessage());