From b1d35b9c2e1dfe18b71ee049375672421e988fd2 Mon Sep 17 00:00:00 2001
From: Vladimir Panfilov <vladimirpanfilov@gmail.com>
Date: Sun, 5 Feb 2017 13:32:52 +0300
Subject: [PATCH] Fix possible endless loop in mxGraphHierarchyModel
 initialRank function (changes ported from 
 mxgraph/javascript/src/js/layout/hierarchical/mxHierarchicalLayout.js)

---
 .../hierarchical/mxHierarchicalLayout.java    | 56 ++++++++++++++++---
 1 file changed, 49 insertions(+), 7 deletions(-)

diff --git a/src/com/mxgraph/layout/hierarchical/mxHierarchicalLayout.java b/src/com/mxgraph/layout/hierarchical/mxHierarchicalLayout.java
index 2b1d2e069..35c609f06 100644
--- a/src/com/mxgraph/layout/hierarchical/mxHierarchicalLayout.java
+++ b/src/com/mxgraph/layout/hierarchical/mxHierarchicalLayout.java
@@ -20,6 +20,7 @@
 import com.mxgraph.layout.hierarchical.stage.mxHierarchicalLayoutStage;
 import com.mxgraph.layout.hierarchical.stage.mxMedianHybridCrossingReduction;
 import com.mxgraph.layout.hierarchical.stage.mxMinimumCycleRemover;
+import com.mxgraph.model.mxCell;
 import com.mxgraph.model.mxGraphModel;
 import com.mxgraph.model.mxIGraphModel;
 import com.mxgraph.view.mxCellState;
@@ -409,13 +410,22 @@ public Set<Object> filterDescendants(Object cell)
 			for (int i = 0; i < childCount; i++)
 			{
 				Object child = model.getChildAt(cell, i);
-				result.addAll(filterDescendants(child));
+
+                // Ignore ports in the layout vertex list, they are dealt with
+                // in the traversal mechanisms
+				if (!isPort((mxCell) child)) {
+					result.addAll(filterDescendants(child));
+				}
 			}
 		}
 
 		return result;
 	}
 
+	private boolean isPort(mxCell cell) {
+		return cell.getGeometry().isRelative();
+	}
+
 	/**
 	 * Traverses the (directed) graph invoking the given function for each
 	 * visited vertex and edge. The function is invoked with the current vertex
@@ -456,19 +466,51 @@ protected void traverse(Object vertex, boolean directed, Object edge,
 
 				int edgeCount = model.getEdgeCount(vertex);
 
+				boolean[] edgeIsSource = new boolean[edgeCount];
+				if (edgeCount > 0) {
+					for (int i = 0; i < edgeCount; i++) {
+						Object e = model.getEdgeAt(vertex, i);
+						edgeIsSource[i] = view.getVisibleTerminal(e, true) == vertex;
+					}
+				}
+
 				if (edgeCount > 0)
 				{
 					for (int i = 0; i < edgeCount; i++)
 					{
 						Object e = model.getEdgeAt(vertex, i);
-						boolean isSource = view.getVisibleTerminal(e, true) == vertex;
 
-						if (!directed || isSource)
+						if (!directed || edgeIsSource[i])
 						{
-							Object next = view.getVisibleTerminal(e, !isSource);
-							traverse(next, directed, e, allVertices,
-									currentComp, hierarchyVertices,
-									filledVertexSet);
+							Object next = view.getVisibleTerminal(e, !edgeIsSource[i]);
+
+                            // Check whether there are more edges incoming from the target vertex than outgoing
+                            // The hierarchical model treats bi-directional parallel edges as being sourced
+                            // from the more "sourced" terminal. If the directions are equal in number, the direction
+                            // is that of the natural direction from the roots of the layout.
+                            // The checks below are slightly more verbose than need be for performance reasons
+							int netCount = 1;
+
+							for (int j = 0; j < edgeCount; j++) {
+								if (j != i) {
+									boolean isSorce2 = edgeIsSource[j];
+									Object e2 = model.getEdgeAt(vertex, j);
+									Object otherTerm = view.getVisibleTerminal(e2, !isSorce2);
+									if (otherTerm == next) {
+										if (isSorce2) {
+											netCount++;
+										} else {
+											netCount--;
+										}
+									}
+								}
+							}
+
+							if (netCount >= 0){
+								traverse(next, directed, e, allVertices,
+										currentComp, hierarchyVertices,
+										filledVertexSet);
+							}
 						}
 					}
 				}