[JBPM-10187] Sorting resources to avoid deadlock (#2318) (#2356)

* [JBPM-10187] Adding traces for process instance info

* [JBPM-10187] Propagate exceptions different than SessionNotFound

* [JBPM-10187] Handling sessionnotfound in local thread

* [JBPM-10187] Using proper SessionNotFoundException

Drools is using a different SessionNotFoundException that runtime
manager. We are only interested on the persistence one.

* [JBPM-10187] Fully logging marshaling exception

* [JBPM-10187] Gonzalo comments



---------

Co-authored-by: Francisco Javier Tirado Sarti <65240126+fjtirado@users.noreply.github.com>
Co-authored-by: Gonzalo Muñoz <gmunozfe@redhat.com>

jbpm-persistence/jbpm-persistence-jpa/src/main/java/org/jbpm/persistence/JpaProcessPersistenceContext.java

@@ -41,6 +41,7 @@
 public class JpaProcessPersistenceContext extends JpaPersistenceContext
     implements
     ProcessPersistenceContext {
+
 
     public JpaProcessPersistenceContext(EntityManager em, TransactionManager txm) {
         super( em, txm );
@@ -66,7 +67,10 @@ public PersistentProcessInstance findProcessInstanceInfo(Long processId) {
     	if( this.pessimisticLocking ) { 
             return em.find( ProcessInstanceInfo.class, processId, lockMode );
         }
-        return em.find( ProcessInstanceInfo.class, processId );
+        logger.trace("Reading process instance info {} with em {}", processId, em);
+        ProcessInstanceInfo pi = em.find(ProcessInstanceInfo.class, processId);
+        logger.trace("Process instance info read {}", pi);
+        return pi;
     }
 
     public void remove(PersistentProcessInstance processInstanceInfo) {

jbpm-persistence/jbpm-persistence-jpa/src/main/java/org/jbpm/persistence/processinstance/ProcessInstanceInfo.java

@@ -55,11 +55,14 @@
 import org.jbpm.workflow.instance.impl.WorkflowProcessInstanceImpl;
 import org.kie.api.runtime.Environment;
 import org.kie.api.runtime.process.ProcessInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Entity
 @SequenceGenerator(name="processInstanceInfoIdSeq", sequenceName="PROCESS_INSTANCE_INFO_ID_SEQ")
 public class ProcessInstanceInfo implements PersistentProcessInstance {
 
+    private static final Logger logger = LoggerFactory.getLogger(ProcessInstanceInfo.class);
     @Id
     @GeneratedValue(strategy = GenerationType.AUTO, generator="processInstanceInfoIdSeq")
     @Column(name = "InstanceId")
@@ -192,9 +195,10 @@ public ProcessInstance getProcessInstance(InternalKnowledgeRuntime kruntime,
                 }
                 context.close();
             } catch ( IOException e ) {
-                e.printStackTrace();
-                throw new IllegalArgumentException( "IOException while loading process instance: " + e.getMessage(),
-                                                    e );
+                throw new IllegalArgumentException( "IOException while loading process instance: " + e.getMessage(), e);
+            } catch (RuntimeException e) {
+                logger.error("Error unmarshalling process instance info {}", processInstanceId, e);
+                throw e;
             }
         }
         ((WorkflowProcessInstanceImpl) processInstance).internalSetStartDate(this.startDate);

jbpm-runtime-manager/src/main/java/org/jbpm/runtime/manager/impl/AbstractRuntimeManager.java

@@ -83,6 +83,7 @@
 import org.kie.internal.runtime.manager.RuntimeManagerRegistry;
 import org.kie.internal.runtime.manager.SecurityManager;
 import org.kie.internal.runtime.manager.SessionFactory;
+import org.kie.internal.runtime.manager.SessionNotFoundException;
 import org.kie.internal.runtime.manager.TaskServiceFactory;
 import org.kie.internal.runtime.manager.context.ProcessInstanceIdContext;
 import org.kie.internal.runtime.manager.deploy.DeploymentDescriptorManager;
@@ -389,8 +390,8 @@ protected boolean canDispose(RuntimeEngine runtime) {
                     && tm.getStatus() != TransactionManager.STATUS_COMMITTED) {
                 return false;
             }
-        } catch (Exception e) {
-            logger.warn("Exception dealing with transaction", e);
+        } catch (SessionNotFoundException e) {
+            logger.warn("Session not found exception", e);
         }        
         return true;
     }

jbpm-runtime-manager/src/main/java/org/jbpm/runtime/manager/impl/PerProcessInstanceRuntimeManager.java

@@ -16,9 +16,12 @@
 package org.jbpm.runtime.manager.impl;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
+import java.util.Set;
 
 import org.drools.core.command.SingleSessionCommandService;
 import org.drools.core.command.impl.CommandBasedStatefulKnowledgeSession;
@@ -191,15 +194,41 @@ public void signalEvent(String type, Object event) {
 
         // process currently active runtime engines
         Map<Object, RuntimeEngine> currentlyActive = local.get();
+
         if (currentlyActive != null && !currentlyActive.isEmpty()) {
-            RuntimeEngine[] activeEngines = currentlyActive.values().toArray(new RuntimeEngine[currentlyActive.size()]);
-            for (RuntimeEngine engine : activeEngines) {
-                Context<?> context = ((RuntimeEngineImpl) engine).getContext();
+            @SuppressWarnings("unchecked")
+            Entry<Object, RuntimeEngine> activeEngines[] = currentlyActive.entrySet()
+                    .toArray(new Entry[currentlyActive.size()]);
+            Set<Object> enginesToDelete = new HashSet<>();
+            for (Entry<Object, RuntimeEngine> engine : activeEngines) {
+                RuntimeEngineImpl engineImpl = (RuntimeEngineImpl) engine.getValue();
+                if (engineImpl==null) {
+                     continue;
+                }
+                if (engineImpl.isDisposed() || engineImpl.isInvalid()) {
+                    Object engineKey = engine.getKey();
+                    logger.trace("Engine with key {} is not longer valid", engineKey);
+                    enginesToDelete.add(engineKey);
+                    continue;
+                }
+                Context<?> context = engineImpl.getContext();
                 if (context != null && context instanceof ProcessInstanceIdContext 
                         && ((ProcessInstanceIdContext) context).getContextId() != null) {
-                    engine.getKieSession().signalEvent(type, event, ((ProcessInstanceIdContext) context).getContextId());
+                    try {
+                        engineImpl.getKieSession().signalEvent(type, event,
+                                ((ProcessInstanceIdContext) context).getContextId());
+                    } catch (org.drools.persistence.api.SessionNotFoundException ex) {
+                        logger.warn(
+                                "Signal event cannot proceed because of session not found exception {} for engine {}",
+                                ex.getMessage(), engineImpl.getKieSessionId());
+                        enginesToDelete.add(engine.getKey());
+                    }
                 }
             }
+            if (!enginesToDelete.isEmpty()) {
+
+                currentlyActive.keySet().removeAll(enginesToDelete);
+            }
         }
     }