Integration of closure into midPoint (mainly changing isAnySubordinat…

…e and Org filter processing).

gui/admin-gui/src/main/resources/initial-objects/020-system-configuration.xml

@@ -30,6 +30,14 @@
             <level>OFF</level>
             <package>org.hibernate.engine.jdbc.spi.SqlExceptionHelper</package>
         </classLogger>
+        <!-- Disabled because we treat locking-related exceptions in the repository.
+             Otherwise the log is filled-in with (innocent but ugly-looking) messages like
+             "ERROR (o.h.engine.jdbc.batch.internal.BatchingBatch): HHH000315: Exception executing batch [Deadlock detected.
+             The current transaction was rolled back." -->
+        <classLogger>
+            <level>OFF</level>
+            <package>org.hibernate.engine.jdbc.batch.internal.BatchingBatch</package>
+        </classLogger>
         <classLogger>
             <!-- disabled because of MID-1612, jasper library needs to be fixed -->
             <level>OFF</level>

gui/admin-gui/src/main/resources/logback.xml

@@ -55,6 +55,7 @@
     <!-- Appender for profiling purposes -->
 
     <logger name="org.hibernate.engine.jdbc.spi.SqlExceptionHelper" level="OFF"/>
+    <logger name="org.hibernate.engine.jdbc.batch.internal.BatchingBatch" level="OFF"/>
     <logger name="PROFILING" level="INFO"/>
     <logger name="com.evolveum.midpoint" level="INFO" />
     <logger name="com.evolveum.midpoint.web.util.MidPointProfilingServletFilter" level="TRACE">

gui/admin-gui/src/test/resources/logback-test.xml

@@ -23,6 +23,7 @@
 		</encoder>
 	</appender>
     <logger name="org.hibernate.engine.jdbc.spi.SqlExceptionHelper" level="OFF"/>
+    <logger name="org.hibernate.engine.jdbc.batch.internal.BatchingBatch" level="OFF"/>
 	<logger name="com.evolveum.midpoint.web" level="TRACE" />
 	<logger name="PROFILING" level="TRACE" />
 	<root level="INFO">

infra/prism/src/main/java/com/evolveum/midpoint/prism/query/OrgFilter.java

@@ -74,7 +74,7 @@ public void setScope(Scope scope) {
         this.scope = scope;
     }
 
-    public void setRoot(boolean root) {
+    private void setRoot(boolean root) {
         this.root = root;
     }
 
@@ -84,7 +84,11 @@ public boolean isRoot() {
 
     @Override
     public OrgFilter clone() {
-        return new OrgFilter(getOrgRef(), getScope());
+        if (isRoot()) {
+            return createRootOrg();
+        } else {
+            return new OrgFilter(getOrgRef(), getScope());
+        }
     }
 
     @Override
@@ -128,6 +132,9 @@ public String debugDump(int indent) {
         StringBuilder sb = new StringBuilder();
         DebugUtil.indentDebugDump(sb, indent);
         sb.append("ORG: \n");
+        if (isRoot()) {
+            sb.append(getOrgRef().debugDump(indent + 1)).append("ROOT\n");
+        }
         if (getOrgRef() != null) {
             sb.append(getOrgRef().debugDump(indent + 1));
             sb.append("\n");

repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/OrgStructTest.java

@@ -137,12 +137,12 @@ public void test001addOrgStructObjects() throws Exception {
         Session session = open();
         try {
             Query qCount = session.createQuery("select count(*) from ROrgClosure");
-            assertCount(qCount, 52);
+            assertCount(qCount, 19);
 
             // check descendants for F0001 org unit
             qCount = session.createQuery("select count(*) from ROrgClosure where ancestorOid = :ancestorOid");
             qCount.setParameter("ancestorOid", ORG_F001_OID);
-            assertCount(qCount, 12);
+            assertCount(qCount, 6);
 
             qCount = session.createQuery("select count(*) from ROrgClosure where ancestorOid = :ancestorOid and descendantOid = :descendantOid");
             qCount.setParameter("ancestorOid", ORG_F001_OID);
@@ -254,15 +254,15 @@ public void test001addOrgStructObjectsIncorrect() throws Exception {
             for (ROrgClosure c : orgClosure) {
                 LOGGER.info("{}", c.getDescendant());
             }
-            AssertJUnit.assertEquals(5, orgClosure.size());
+            AssertJUnit.assertEquals(4, orgClosure.size());
 
             criteria = session.createCriteria(ROrgClosure.class)
                     .createCriteria("ancestor", "anc")
                     .setFetchMode("ancestor", FetchMode.JOIN)
                     .add(Restrictions.eq("anc.oid", ORG_F009_OID));
 
             orgClosure = criteria.list();
-            AssertJUnit.assertEquals(4, orgClosure.size());
+            AssertJUnit.assertEquals(3, orgClosure.size());
 
 
         ObjectQuery query = new ObjectQuery();
@@ -276,7 +276,7 @@ public void test001addOrgStructObjectsIncorrect() throws Exception {
             UserType elaine1 = users.get(0).asObjectable();
             LOGGER.info("--->elaine1<----");
 
-            AssertJUnit.assertEquals("Expected name elaine, but got " + elaine1.getName().getOrig(), "elaine1", elaine1.getName().getOrig());
+            AssertJUnit.assertEquals("Expected name elaine1, but got " + elaine1.getName().getOrig(), "elaine1", elaine1.getName().getOrig());
             AssertJUnit.assertEquals("Expected elaine has one org ref, but got " + elaine1.getParentOrgRef().size(), 2, elaine1.getParentOrgRef().size());
             AssertJUnit.assertEquals("Parent org ref oid not equal.", "00000000-8888-6666-0000-100000000011", elaine1.getParentOrgRef().get(0).getOid());
         } finally {
@@ -457,7 +457,7 @@ public void test003modifyOrgStructDeleteRefIncorrect() throws Exception {
         }
     }
 
-    @Test
+    @Test(enabled = false)          // users are not in the closure any more
     public void test004modifyOrgStructAddUser() throws Exception {
         Session session = open();
         try {
@@ -533,7 +533,7 @@ public void test007searchOrgStructOrgDepth() throws Exception {
                 LOGGER.info("=> A: {}, D: {}", new Object[]{o.getAncestor().toJAXB(prismContext, null),
                         o.getDescendant().toJAXB(prismContext, null)});
             }
-            AssertJUnit.assertEquals(4, orgClosure.size());
+            AssertJUnit.assertEquals(1, orgClosure.size());
             session.getTransaction().commit();
             session.close();
 
@@ -547,7 +547,9 @@ public void test007searchOrgStructOrgDepth() throws Exception {
             }
             AssertJUnit.assertEquals(4, sOrgClosure.size());
         } finally {
-            close(session);
+            if (session.isOpen()) {
+                close(session);
+            }
         }
     }
 

repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/AbstractOrgClosureTest.java

@@ -108,12 +108,14 @@ protected Session getSession() {
     }
 
     protected void checkClosure(Set<String> oidsToCheck) {
+        boolean matrixProblem = false;
         if (getConfiguration().isCheckClosureMatrix()) {
-            checkClosureMatrix();
+            matrixProblem = checkClosureMatrix();
         }
         if (getConfiguration().isCheckChildrenSets()) {
             checkChildrenSets(oidsToCheck);
         }
+        assertFalse("A difference in transitive closure matrix was detected", matrixProblem);
     }
 
     protected void checkClosureUnconditional(Set<String> oidsToCheck) {
@@ -151,9 +153,9 @@ private void checkChildrenSets(Set<String> oidsToCheck) {
     /**
      * Recomputes closure table from scratch (using matrix multiplication) and compares it with M_ORG_CLOSURE.
      */
-    private static final boolean DUMP_TC_MATRIX_DETAILS = false;
+    private static final boolean DUMP_TC_MATRIX_DETAILS = true;
 
-    protected void checkClosureMatrix() {
+    protected boolean checkClosureMatrix() {
         Session session = getSession();
         // we compute the closure table "by hand" as 1 + A + A^2 + A^3 + ... + A^n where n is the greatest expected path length
         int vertices = getVertices().size();
@@ -225,13 +227,14 @@ protected void checkClosureMatrix() {
             for (int j = 0; j < vertices; j++) {
                 double delta = result.get(i, j);
                 if (Math.round(delta) != 0) {
-                    System.err.println("delta("+vertexList.get(i)+","+vertexList.get(j)+") = " + delta);
+                    System.err.println("delta("+vertexList.get(i)+","+vertexList.get(j)+") = " + delta +
+                            " (closureInDB=" + closureInDatabase.get(i, j) + ", expected=" + (result.get(i, j) + closureInDatabase.get(i, j)) + ")");
                     LOGGER.error("delta("+vertexList.get(i)+","+vertexList.get(j)+") = " + delta);
                     problem = true;
                 }
             }
         }
-        assertFalse("Difference found", problem);
+        return problem;
     }
 
     protected Set<String> getActualChildrenOf(String ancestor) {

repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureConcurrencyTest.java

@@ -51,11 +51,12 @@ public class OrgClosureConcurrencyTest extends AbstractOrgClosureTest {
     private static final Trace LOGGER = TraceManager.getTrace(OrgClosureConcurrencyTest.class);
 
     private static final int[] ORG_CHILDREN_IN_LEVEL  = { 5, 3, 3, 3, 0  };
-    private static final int[] USER_CHILDREN_IN_LEVEL = { 0, 1, 2, 3, 3  };
-    private static final int[] PARENTS_IN_LEVEL       = { 0, 1, 2, 3, 3  };
+    private static final int[] USER_CHILDREN_IN_LEVEL = { 0, 2, 2, 3, 3  };
+    private static final int[] PARENTS_IN_LEVEL       = { 0, 2, 2, 3, 3  };
     private static final int[] LINK_ROUNDS_FOR_LEVELS = { 0, 15, 20, 30  };
     private static final int[] NODE_ROUNDS_FOR_LEVELS = { 3, 15, 20, 30  };
     private static final int[] USER_ROUNDS_FOR_LEVELS = { 0, 5 ,5, 10 };
+    public static final int THREADS = 3;
 
     private OrgClosureTestConfiguration configuration;
 
@@ -136,7 +137,7 @@ protected void _test200AddRemoveLinksMT(final boolean random) throws Exception {
             }
         }
 
-        int numberOfRunners = 3;
+        int numberOfRunners = THREADS;
         info("Edges to remove/add (" + edgesToRemove.size() + ": " + edgesToRemove);
         info("Number of runners: " + numberOfRunners);
         final List<Thread> runners = Collections.synchronizedList(new ArrayList<Thread>());
@@ -294,7 +295,7 @@ protected void _test300AddRemoveNodesMT(final boolean random) throws Exception {
             generateNodesAtOneLevel(nodesToRemove, nodesToAdd, OrgType.class, rounds, levelOids, opResult);
         }
 
-        int numberOfRunners = 3;
+        int numberOfRunners = THREADS;
         final List<Thread> runners = Collections.synchronizedList(new ArrayList<Thread>());
 
         for (int i = 0; i < numberOfRunners; i++) {

repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureCorrectnessTest.java

@@ -27,7 +27,7 @@
 @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
 public class OrgClosureCorrectnessTest extends AbstractOrgClosureTest {
 
-    private static final int[] ORG_CHILDREN_IN_LEVEL  = { 1, 2, 3  };
+    private static final int[] ORG_CHILDREN_IN_LEVEL  = { 4, 3, 3  };
     private static final int[] USER_CHILDREN_IN_LEVEL = null;
     private static final int[] PARENTS_IN_LEVEL       = { 0, 2, 2  };
     private static final int[] LINK_ROUNDS_FOR_LEVELS = { 0, 5, 10 };

repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosurePerformanceTest1.java

@@ -16,11 +16,15 @@
 
 package com.evolveum.midpoint.repo.sql.closure;
 
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.ContextConfiguration;
 import org.testng.annotations.Test;
 
 /**
  * @author mederly
  */
+@ContextConfiguration(locations = {"../../../../../../ctx-test.xml"})
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
 public class OrgClosurePerformanceTest1 extends AbstractOrgClosureTest {
 
     // relatively smaller graph (OrgClosure has cca 12K rows)

repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosurePerformanceTest2.java

@@ -16,20 +16,23 @@
 
 package com.evolveum.midpoint.repo.sql.closure;
 
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.ContextConfiguration;
 import org.testng.annotations.Test;
 
 /**
  * @author mederly
  */
+@ContextConfiguration(locations = {"../../../../../../ctx-test.xml"})
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
 public class OrgClosurePerformanceTest2 extends AbstractOrgClosureTest {
 
     // relatively bigger graph
-    private static final int[] ORG_CHILDREN_IN_LEVEL  = { 1, 5, 3, 3, 5, 4,  0};
-    private static final int[] USER_CHILDREN_IN_LEVEL = { 0, 3, 4, 5, 6, 7, 10};
-    private static final int[] PARENTS_IN_LEVEL       = { 0, 1, 2, 2, 2, 2,  2};
-    private static final int[] LINK_ROUNDS_FOR_LEVELS = { 0, 10, 15,15,15,15, 0 };
-    private static final int[] NODE_ROUNDS_FOR_LEVELS = { 5, 10, 15,15,15,15, 0 };
-    private static final int[] USER_ROUNDS_FOR_LEVELS = { 0, 10,10,20,20,20, 20};
+    private static final int[] ORG_CHILDREN_IN_LEVEL  = { 2, 5, 3, 3, 5, 4 };
+    private static final int[] USER_CHILDREN_IN_LEVEL = null;
+    private static final int[] PARENTS_IN_LEVEL       = { 0, 1, 2, 2, 2, 2 };
+    private static final int[] LINK_ROUNDS_FOR_LEVELS = { 0, 10, 15,15,15,15 };
+    private static final int[] NODE_ROUNDS_FOR_LEVELS = { 5, 10, 15,15,15,15 };
 
     private OrgClosureTestConfiguration configuration;
 

repo/repo-sql-impl-test/testng.xml

@@ -17,42 +17,43 @@
   -->
 <!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
 <suite name="unit" parallel="false">
-    <!--<test name="Startup Tests" parallel="false" preserve-order="true" verbose="10">-->
-        <!--<classes>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.SpringApplicationContextTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.EmbeddedServerModeTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.DataSourceTest"/>-->
-        <!--</classes>-->
-    <!--</test>-->
+    <test name="Startup Tests" parallel="false" preserve-order="true" verbose="10">
+        <classes>
+            <class name="com.evolveum.midpoint.repo.sql.SpringApplicationContextTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.EmbeddedServerModeTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.DataSourceTest"/>
+        </classes>
+    </test>
     <test name="Query Add Tests" parallel="false" preserve-order="true" verbose="10" enabled="true">
         <classes>
-            <!--<class name="com.evolveum.midpoint.repo.sql.QueryInterpreterTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.DeleteTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.AddGetObjectTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.AddOverwriteTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.EncodingTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.ModifyTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.ModifyUser"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.ModifyAssignmentTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.ResourceModifyTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.ListAccountShadowOwnerTest"/>-->
+            <class name="com.evolveum.midpoint.repo.sql.QueryInterpreterTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.DeleteTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.AddGetObjectTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.AddOverwriteTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.EncodingTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.ModifyTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.ModifyUser"/>
+            <class name="com.evolveum.midpoint.repo.sql.ModifyAssignmentTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.ResourceModifyTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.ListAccountShadowOwnerTest"/>
             <class name="com.evolveum.midpoint.repo.sql.OrgStructTest"/>
-            <!--<class name="com.evolveum.midpoint.repo.sql.closure.OrgClosureSimpleTreeTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.SearchTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.CleanupTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.SearchShadowOwnerTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.PerformanceTest"/>-->
+            <class name="com.evolveum.midpoint.repo.sql.closure.OrgClosureCorrectnessTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.SearchTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.CleanupTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.SearchShadowOwnerTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.PerformanceTest"/>
+        </classes>
+    </test>
+    <test name="Concurrency Tests" parallel="false" preserve-order="true" verbose="10" enabled="true">
+        <classes>
+            <class name="com.evolveum.midpoint.repo.sql.ConcurrencyTest"/>
+            <!--<class name="com.evolveum.midpoint.repo.sql.closure.OrgClosureConcurrencyTest"/>-->
+        </classes>
+    </test>
+    <test name="Utils Tests" parallel="false" preserve-order="true" verbose="10" enabled="true">
+        <classes>
+            <class name="com.evolveum.midpoint.repo.sql.RAnyConverterStaticTest"/>
+            <class name="com.evolveum.midpoint.repo.sql.RUtilTest"/>
         </classes>
     </test>
-    <!--<test name="Concurrency Tests" parallel="false" preserve-order="true" verbose="10" enabled="true">-->
-        <!--<classes>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.ConcurrencyTest"/>-->
-        <!--</classes>-->
-    <!--</test>-->
-    <!--<test name="Utils Tests" parallel="false" preserve-order="true" verbose="10" enabled="true">-->
-        <!--<classes>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.RAnyConverterStaticTest"/>-->
-            <!--<class name="com.evolveum.midpoint.repo.sql.RUtilTest"/>-->
-        <!--</classes>-->
-    <!--</test>-->
 </suite>