diff --git a/repo/repo-cache/src/main/java/com/evolveum/midpoint/repo/cache/RepositoryCache.java b/repo/repo-cache/src/main/java/com/evolveum/midpoint/repo/cache/RepositoryCache.java index fc92c71f628..9546a1c8cf9 100644 --- a/repo/repo-cache/src/main/java/com/evolveum/midpoint/repo/cache/RepositoryCache.java +++ b/repo/repo-cache/src/main/java/com/evolveum/midpoint/repo/cache/RepositoryCache.java @@ -298,8 +298,8 @@ public RepositoryDiag getRepositoryDiag() { public void repositorySelfTest(OperationResult parentResult) { repository.repositorySelfTest(parentResult); } - - private void cacheObject(Map> cache, PrismObject object) { + + private void cacheObject(Map> cache, PrismObject object) { if (cache != null) { cache.put(object.getOid(), (PrismObject) object.clone()); } diff --git a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/AbstractOrgClosureTest.java b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/AbstractOrgClosureTest.java index 678ca67b44d..5d5edd94e8b 100644 --- a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/AbstractOrgClosureTest.java +++ b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/AbstractOrgClosureTest.java @@ -150,6 +150,8 @@ private void checkChildrenSets(Set oidsToCheck) { /** * Recomputes closure table from scratch (using matrix multiplication) and compares it with M_ORG_CLOSURE. */ + private static final boolean DUMP_TC_MATRIX_DETAILS = false; + protected void checkClosureMatrix() { Session session = getSession(); // we compute the closure table "by hand" as 1 + A + A^2 + A^3 + ... + A^n where n is the greatest expected path length @@ -159,6 +161,9 @@ protected void checkClosureMatrix() { // used to give indices to vertices List vertexList = new ArrayList<>(getVertices()); + + if (DUMP_TC_MATRIX_DETAILS) LOGGER.info("Vertex list = {}", vertexList); + DoubleMatrix2D a = new SparseDoubleMatrix2D(vertices, vertices); // for (int i = 0; i < vertices; i++) { // a.setQuick(i, i, 1.0); @@ -184,6 +189,8 @@ protected void checkClosureMatrix() { } LOGGER.info("TC matrix computed in {} ms", System.currentTimeMillis() - start); + if (DUMP_TC_MATRIX_DETAILS) LOGGER.info("TC matrix expected = {}", result); + Query q = session.createSQLQuery("select descendant_oid, ancestor_oid, val from m_org_closure") .addScalar("descendant_oid", StringType.INSTANCE) .addScalar("ancestor_oid", StringType.INSTANCE) @@ -193,17 +200,25 @@ protected void checkClosureMatrix() { DoubleMatrix2D closureInDatabase = new SparseDoubleMatrix2D(vertices, vertices); for (Object[] item : list) { + int val = Integer.parseInt(item[2].toString()); + if (val == 0) { + throw new IllegalStateException("Row with val == 0 in closure table: " + list); + } closureInDatabase.set(vertexList.indexOf(item[0]), vertexList.indexOf(item[1]), - Integer.parseInt(item[2].toString())); + val); } + if (DUMP_TC_MATRIX_DETAILS) LOGGER.info("TC matrix fetched from db = {}", closureInDatabase); + double zSumResultBefore = result.zSum(); double zSumClosureInDb = closureInDatabase.zSum(); result.assign(closureInDatabase, Functions.minus); double zSumResultAfter = result.zSum(); LOGGER.info("Summary of items in closure computed: {}, in DB-stored closure: {}, delta: {}", new Object[]{zSumResultBefore, zSumClosureInDb, zSumResultAfter}); + if (DUMP_TC_MATRIX_DETAILS) LOGGER.info("Difference matrix = {}", result); + boolean problem = false; for (int i = 0; i < vertices; i++) { for (int j = 0; j < vertices; j++) { diff --git a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureConcurrencyTest.java b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureConcurrencyTest.java index 011cdd8b1ec..037d6b2cb2f 100644 --- a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureConcurrencyTest.java +++ b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureConcurrencyTest.java @@ -27,6 +27,7 @@ import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType; import com.evolveum.midpoint.xml.ns._public.common.common_3.OrgType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.UserType; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.testng.annotations.Test; @@ -80,8 +81,8 @@ public OrgClosureTestConfiguration getConfiguration() { @Test(enabled = true) public void test150CheckClosure() throws Exception { _test150CheckClosure(); } @Test(enabled = true) public void test200AddRemoveLinksSeq() throws Exception { _test200AddRemoveLinksMT(false); } @Test(enabled = true) public void test201AddRemoveLinksRandom() throws Exception { _test200AddRemoveLinksMT(true); } - @Test(enabled = true) public void test300AddRemoveOrgsSeq() throws Exception { _test300AddRemoveOrgsMT(false); } - @Test(enabled = true) public void test301AddRemoveOrgsRandom() throws Exception { _test300AddRemoveOrgsMT(true); } + @Test(enabled = true) public void test300AddRemoveNodesSeq() throws Exception { _test300AddRemoveNodesMT(false); } + @Test(enabled = true) public void test301AddRemoveNodesRandom() throws Exception { _test300AddRemoveNodesMT(true); } //@Test(enabled = true) public void test310AddRemoveUsers() throws Exception { _test310AddRemoveUsersMT(); } /** @@ -154,7 +155,9 @@ public void run() { LOGGER.info("Removing {}", edge); removeEdge(edge); info(Thread.currentThread().getName() + " removed " + edge); - edgesToRemove.remove(edge); + synchronized (OrgClosureConcurrencyTest.this) { + edgesToRemove.remove(edge); + } } } catch (Throwable e) { e.printStackTrace(); @@ -197,7 +200,9 @@ public void run() { LOGGER.info("Adding {}", edge); addEdge(edge); info(Thread.currentThread().getName() + " re-added " + edge); - edgesToAdd.remove(edge); + synchronized (OrgClosureConcurrencyTest.this) { + edgesToAdd.remove(edge); + } } } catch (Throwable e) { e.printStackTrace(); @@ -274,41 +279,25 @@ private void addEdge(Edge edge) throws ObjectAlreadyExistsException, ObjectNotFo } } - protected void _test300AddRemoveOrgsMT(final boolean random) throws Exception { - OperationResult opResult = new OperationResult("===[ test300AddRemoveOrgsMT ]==="); + protected void _test300AddRemoveNodesMT(final boolean random) throws Exception { + OperationResult opResult = new OperationResult("===[ test300AddRemoveNodesMT ]==="); - info("test300AddRemoveOrgs starting with random = " + random); + info("test300AddRemoveNodes starting with random = " + random); - final Set orgsToRemove = Collections.synchronizedSet(new HashSet()); - final Set orgsToAdd = Collections.synchronizedSet(new HashSet()); + final Set nodesToRemove = Collections.synchronizedSet(new HashSet()); + final Set nodesToAdd = Collections.synchronizedSet(new HashSet()); final List exceptions = Collections.synchronizedList(new ArrayList()); for (int level = 0; level < getConfiguration().getNodeRoundsForLevel().length; level++) { int rounds = getConfiguration().getNodeRoundsForLevel()[level]; List levelOids = orgsByLevels.get(level); - if (levelOids.isEmpty()) { - continue; - } - int retries = 0; - for (int round = 0; round < rounds; round++) { - - int index = (int) Math.floor(Math.random() * levelOids.size()); - String oid = levelOids.get(index); - OrgType org = repositoryService.getObject(OrgType.class, oid, null, opResult).asObjectable(); - - if (orgsToRemove.contains(org)) { - round--; - if (++retries == 1000) { - throw new IllegalStateException("Too many retries"); // primitive attempt to break potential cycles when there is not enough edges to process - } else { - continue; - } - } - - orgsToRemove.add(org); - orgsToAdd.add(org); - } + generateNodesAtOneLevel(nodesToRemove, nodesToAdd, OrgType.class, rounds, levelOids, opResult); + } + for (int level = 0; level < getConfiguration().getUserRoundsForLevel().length; level++) { + int rounds = getConfiguration().getUserRoundsForLevel()[level]; + List levelOids = usersByLevels.get(level); + generateNodesAtOneLevel(nodesToRemove, nodesToAdd, UserType.class, rounds, levelOids, opResult); } int numberOfRunners = 3; @@ -320,18 +309,20 @@ protected void _test300AddRemoveOrgsMT(final boolean random) throws Exception { public void run() { try { while (true) { - OrgType org = getNext(orgsToRemove, random); - if (org == null) { + ObjectType objectType = getNext(nodesToRemove, random); + if (objectType == null) { break; } - LOGGER.info("Removing {}", org); + LOGGER.info("Removing {}", objectType); try { - removeObject(org); - orgsToRemove.remove(org); - info(Thread.currentThread().getName() + " removed " + org); + removeObject(objectType); + synchronized (OrgClosureConcurrencyTest.this) { + nodesToRemove.remove(objectType); + } + info(Thread.currentThread().getName() + " removed " + objectType); } catch (ObjectNotFoundException e) { // this is OK - info(Thread.currentThread().getName() + ": " + org + " already deleted"); + info(Thread.currentThread().getName() + ": " + objectType + " already deleted"); } } } catch (Throwable e) { @@ -351,8 +342,8 @@ public void run() { Thread.sleep(100); // primitive way of waiting } - if (!orgsToRemove.isEmpty()) { - throw new AssertionError("Orgs to remove is not empty, see the console or log: " + orgsToRemove); + if (!nodesToRemove.isEmpty()) { + throw new AssertionError("Nodes to remove is not empty, see the console or log: " + nodesToRemove); } if (!exceptions.isEmpty()) { @@ -368,18 +359,20 @@ public void run() { public void run() { try { while (true) { - OrgType org = getNext(orgsToAdd, random); - if (org == null) { + ObjectType objectType = getNext(nodesToAdd, random); + if (objectType == null) { break; } - LOGGER.info("Adding {}", org); + LOGGER.info("Adding {}", objectType); try { - addObject(org.clone()); - orgsToAdd.remove(org); - info(Thread.currentThread().getName() + " re-added " + org); + addObject(objectType.clone()); + synchronized (OrgClosureConcurrencyTest.this) { + nodesToAdd.remove(objectType); + } + info(Thread.currentThread().getName() + " re-added " + objectType); } catch (ObjectAlreadyExistsException e) { // this is OK - info(Thread.currentThread().getName() + ": " + org + " already exists"); + info(Thread.currentThread().getName() + ": " + objectType + " already exists"); } } } catch (Throwable e) { @@ -399,8 +392,8 @@ public void run() { Thread.sleep(100); // primitive way of waiting } - if (!orgsToAdd.isEmpty()) { - throw new AssertionError("Orgs to add is not empty, see the console or log: " + orgsToAdd); + if (!nodesToAdd.isEmpty()) { + throw new AssertionError("Nodes to add is not empty, see the console or log: " + nodesToAdd); } if (!exceptions.isEmpty()) { @@ -411,6 +404,34 @@ public void run() { info("Consistency after re-adding OK"); } + private void generateNodesAtOneLevel(Set nodesToRemove, Set nodesToAdd, + Class clazz, + int rounds, List candidateOids, + OperationResult opResult) throws ObjectNotFoundException, SchemaException { + if (candidateOids.isEmpty()) { + return; + } + int retries = 0; + for (int round = 0; round < rounds; round++) { + + int index = (int) Math.floor(Math.random() * candidateOids.size()); + String oid = candidateOids.get(index); + ObjectType objectType = repositoryService.getObject(clazz, oid, null, opResult).asObjectable(); + + if (nodesToRemove.contains(objectType)) { + round--; + if (++retries == 1000) { + throw new IllegalStateException("Too many retries"); // primitive attempt to break potential cycles when there is not enough edges to process + } else { + continue; + } + } + + nodesToRemove.add(objectType); + nodesToAdd.add(objectType); + } + } + void removeObject(ObjectType objectType) throws Exception { repositoryService.deleteObject(objectType.getClass(), objectType.getOid(), new OperationResult("dummy")); synchronized(orgGraph) { diff --git a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureCorrectnessTest.java b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureCorrectnessTest.java index 530af1ce428..5e9402df4a0 100644 --- a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureCorrectnessTest.java +++ b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureCorrectnessTest.java @@ -27,13 +27,21 @@ @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) public class OrgClosureCorrectnessTest extends AbstractOrgClosureTest { - private static final int[] ORG_CHILDREN_IN_LEVEL = { 3, 2, 3, 0 }; + private static final int[] ORG_CHILDREN_IN_LEVEL = { 1, 2, 3, 0 }; private static final int[] USER_CHILDREN_IN_LEVEL = { 0, 1, 2, 3 }; private static final int[] PARENTS_IN_LEVEL = { 0, 2, 2, 3 }; private static final int[] LINK_ROUNDS_FOR_LEVELS = { 0, 5, 10 }; private static final int[] NODE_ROUNDS_FOR_LEVELS = { 1, 5, 10 }; private static final int[] USER_ROUNDS_FOR_LEVELS = { 0, 5 ,5, 10 }; + // trivial case for debugging +// private static final int[] ORG_CHILDREN_IN_LEVEL = { 1, 2, 0 }; +// private static final int[] USER_CHILDREN_IN_LEVEL = { 0, 0, 0 }; +// private static final int[] PARENTS_IN_LEVEL = { 0, 1, 1 }; +// private static final int[] LINK_ROUNDS_FOR_LEVELS = { 0, 1, 0 }; +// private static final int[] NODE_ROUNDS_FOR_LEVELS = { 1, 5, 10 }; +// private static final int[] USER_ROUNDS_FOR_LEVELS = { 0, 5 ,5, 10 }; + private OrgClosureTestConfiguration configuration; public OrgClosureCorrectnessTest() { diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/OrgClosureManager.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/OrgClosureManager.java index 6f68684c001..54b3b344ee5 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/OrgClosureManager.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/OrgClosureManager.java @@ -19,15 +19,25 @@ import com.evolveum.midpoint.prism.PrismReferenceValue; import com.evolveum.midpoint.prism.delta.ItemDelta; import com.evolveum.midpoint.prism.delta.ReferenceDelta; -import com.evolveum.midpoint.repo.sql.data.common.*; +import com.evolveum.midpoint.prism.query.ObjectQuery; +import com.evolveum.midpoint.repo.sql.data.common.ROrgClosure; +import com.evolveum.midpoint.repo.sql.data.common.other.RObjectType; +import com.evolveum.midpoint.schema.ResultHandler; +import com.evolveum.midpoint.schema.result.OperationResult; +import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.util.exception.SystemException; +import com.evolveum.midpoint.util.logging.LoggingUtils; import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.xml.ns._public.common.common_3.FocusType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType; import com.evolveum.midpoint.xml.ns._public.common.common_3.OrgType; import org.apache.commons.lang.Validate; +import org.apache.commons.lang.mutable.MutableInt; import org.hibernate.Query; import org.hibernate.Session; +import org.hibernate.type.IntegerType; import java.util.*; @@ -128,6 +138,16 @@ public OrgClosureManager(SqlRepositoryConfiguration repoConfiguration) { public void updateOrgClosure(PrismObject originalObject, Collection modifications, Session session, String oid, Class type, Operation operation) { + + switch (repoConfiguration.getOrgClosureObjects()) { + case NONE: + return; + case FOCUS: + if (!FocusType.class.isAssignableFrom(type)) { + return; + } + } + session.flush(); session.clear(); @@ -154,11 +174,126 @@ public void updateOrgClosure(PrismObject handler = new ResultHandler() { + @Override + public boolean handle(PrismObject object, OperationResult parentResult) { + LOGGER.trace("Processing {}", object); + handleAdd(object.getOid(), object.getCompileTimeClass(), getExistingParentOids(object), session); + orgsProcessed.add(1); + int currentState = orgsProcessed.intValue(); + if (currentState % 100 == 0) { + LOGGER.info("{} organizations processed (out of {})", currentState, orgsTotal); + } + return true; + } + }; + service.searchObjectsIterative(OrgType.class, new ObjectQuery(), handler, null, result); + + session.getTransaction().commit(); + LOGGER.info("Org closure table was successfully rebuilt; all {} organizations processed", orgsTotal); + + } catch (SchemaException|RuntimeException e) { + session.getTransaction().rollback(); + LoggingUtils.logException(LOGGER, "Rebuilding closure failed", e); + if (stopOnFailure) { + throw new SystemException("Rebuilding closure failed: " + e.getMessage(), e); + } + } + } + //endregion + //region Handling ADD operation // we can safely expect that the object didn't exist before (because the "overwriting add" is sent to us as MODIFY operation) private void handleAdd(String oid, Class type, List deltas, Session session) { + handleAdd(oid, type, getParentOidsFromAddDeltas(deltas, null), session); + } + // parents may be non-existent at this point + private void handleAdd(String oid, Class type, Set parents, Session session) { // adding self-record session.save(new ROrgClosure(oid, oid, 1)); session.flush(); @@ -173,7 +308,6 @@ private void handleAdd(String oid, Class type, List parents = getParentOidsFromAddDeltas(deltas, null); Collection livingParents = retainExistingOids(parents, session); if (livingParents.size() <= 1 && (livingChildren == null || livingChildren.isEmpty())) { @@ -345,7 +479,7 @@ private void addIndependentEdges(List edges, Session session) { session.flush(); session.clear(); - LOGGER.trace("--------------------- DONE ADD EDGES: {} ({} ms) ----------------", edges, System.currentTimeMillis()-start); + LOGGER.trace("--------------------- DONE ADD EDGES: {} ({} ms) ----------------", edges, System.currentTimeMillis() - start); } private void dropDeltaTableIfNecessary(Session session, String deltaTempTableName) { @@ -402,7 +536,7 @@ private void handleDelete(String oid, Class type, Sess private void handleDeleteLeaf(String oid, Session session) { Query removeFromClosureQuery = session.createSQLQuery( - "delete from "+closureTableName+" " + + "delete from " + closureTableName + " " + "where descendant_oid = :oid"); removeFromClosureQuery.setString("oid", oid); int count = removeFromClosureQuery.executeUpdate(); @@ -421,9 +555,11 @@ private void removeIndependentEdges(List edges, Session session) { String deleteFromClosureQueryText, updateInClosureQueryText; if (isH2()) { - deleteFromClosureQueryText = "delete from " + closureTableName + " " + - "where (descendant_oid, ancestor_oid, val) in " + - "(select (descendant_oid, ancestor_oid, val) from " + deltaTempTableName + ")"; + // delete with join is not supported by H2 + deleteFromClosureQueryText = "delete from " + closureTableName + " cl " + + "where exists (" + + "select 0 from " + deltaTempTableName + " delta " + + "where cl.descendant_oid = delta.descendant_oid and cl.ancestor_oid = delta.ancestor_oid and cl.val = delta.val)"; updateInClosureQueryText = "update " + closureTableName + " " + "set val = val - (select val from " + deltaTempTableName + " td " + "where td.descendant_oid=" + closureTableName + ".descendant_oid and td.ancestor_oid=" + closureTableName + ".ancestor_oid) " + @@ -546,7 +682,17 @@ private List getParents(String oid, Session session) { } private List getChildren(String oid, Session session) { - Query childrenQuery = session.createQuery("select distinct ownerOid from RParentOrgRef where targetOid=:oid"); + Query childrenQuery; + if (repoConfiguration.getOrgClosureObjects() == OrgClosureObjects.ALL) { + childrenQuery = session.createQuery("select distinct ownerOid from RParentOrgRef where targetOid=:oid"); + } else if (repoConfiguration.getOrgClosureObjects() == OrgClosureObjects.FOCUS) { + childrenQuery = session.createQuery("select distinct parentRef.ownerOid from RParentOrgRef as parentRef" + + " join parentRef.owner as owner where parentRef.targetOid=:oid" + + " and owner.objectTypeClass in (:types)"); + childrenQuery.setParameterList("types", Arrays.asList(RObjectType.USER, RObjectType.ORG, RObjectType.ROLE)); + } else { + throw new IllegalStateException("Unknown option: " + repoConfiguration.getOrgClosureObjects()); + } childrenQuery.setString("oid", oid); return childrenQuery.list(); } @@ -612,9 +758,9 @@ private String computeDeltaTable(List edges, Session session) { count = insertQuery.executeUpdate(); } else { String createTablePrefix; - if (isH2()) { + if (isPostgreSQL()) { createTablePrefix = "create cached local temporary " + deltaTempTableName + " on commit drop as "; - } else if (isPostgreSQL()) { + } else if (isH2()) { createTablePrefix = "create temporary table " + deltaTempTableName + " as "; } else if (isMySQL()) { createTablePrefix = "create temporary table " + deltaTempTableName + " engine=memory as "; @@ -664,7 +810,7 @@ private String getWhereClause(List edges) { whereClause.append(" or "); } whereClause.append("(t1.ancestor_oid = '").append(edge.getTail()).append("'"); - whereClause.append("and t2.descendant_oid = '").append(edge.getHead()).append("')"); + whereClause.append(" and t2.descendant_oid = '").append(edge.getHead()).append("')"); } return whereClause.toString(); } @@ -756,4 +902,53 @@ public long getLastOperationDuration() { return lastOperationDuration; } + public static enum StartupAction { + + NONE("none"), CHECK("check"), REBUILD_IF_NEEDED("rebuidIfNeeded"), ALWAYS_REBUILD("alwaysRebuild"); + + private String value; + + StartupAction(String value) { + this.value = value; + } + + @Override + public String toString() { + return value; + } + + public static StartupAction fromValue(String v) { + for (StartupAction a: StartupAction.values()) { + if (a.value.equals(v)) { + return a; + } + } + throw new IllegalArgumentException(v); + } + } + + public static enum OrgClosureObjects { + + NONE("none"), FOCUS("focus"), ALL("all"); + + private String value; + + OrgClosureObjects(String value) { + this.value = value; + } + + @Override + public String toString() { + return value; + } + + public static OrgClosureObjects fromValue(String v) { + for (OrgClosureObjects a: OrgClosureObjects.values()) { + if (a.value.equals(v)) { + return a; + } + } + throw new IllegalArgumentException(v); + } + } } diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlBaseService.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlBaseService.java index 9790d11155b..94b6bc63b29 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlBaseService.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlBaseService.java @@ -180,6 +180,14 @@ private boolean isExceptionRelatedToSerialization(Exception ex) { // todo: so it is probably not very safe to test for codes without testing for specific database (h2, oracle) // but the risk of problem is quite low here, so let it be... + // strange exception occurring in MySQL when doing multithreaded org closure maintenance + // alternatively we might check for error code = 1030, sql state = HY000 + // but that would cover all cases of "Got error XYZ from storage engine" + if (getConfiguration().isUsingMySQL() && sqlException.getMessage() != null && + sqlException.getMessage().contains("Got error -1 from storage engine")) { + return true; + } + return sqlException.getErrorCode() == 50200 || sqlException.getErrorCode() == 40001 || "40001".equals(sqlException.getSQLState()) diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryConfiguration.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryConfiguration.java index d7ea83083c9..d92feba70cd 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryConfiguration.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryConfiguration.java @@ -66,6 +66,11 @@ public class SqlRepositoryConfiguration { public static final String PROPERTY_ITERATIVE_SEARCH_BY_PAGING = "iterativeSearchByPaging"; public static final String PROPERTY_ITERATIVE_SEARCH_BY_PAGING_BATCH_SIZE = "iterativeSearchByPagingBatchSize"; + //closure + public static final String PROPERTY_ORG_CLOSURE_OBJECTS = "orgClosureObjects"; + public static final String PROPERTY_ORG_CLOSURE_STARTUP_ACTION = "orgClosureStartupAction"; + public static final String PROPERTY_STOP_ON_ORG_CLOSURE_STARTUP_FAILURE = "stopOnOrgClosureStartupFailure"; + //embedded configuration private boolean embedded; private boolean asServer; @@ -96,6 +101,10 @@ public class SqlRepositoryConfiguration { private boolean iterativeSearchByPaging; private int iterativeSearchByPagingBatchSize; + private OrgClosureManager.OrgClosureObjects orgClosureObjects; + private OrgClosureManager.StartupAction orgClosureStartupAction; + private boolean stopOnOrgClosureStartupFailure; + public SqlRepositoryConfiguration(Configuration configuration) { setAsServer(configuration.getBoolean(PROPERTY_AS_SERVER, false)); setBaseDir(configuration.getString(PROPERTY_BASE_DIR, baseDir)); @@ -128,6 +137,10 @@ public SqlRepositoryConfiguration(Configuration configuration) { setIterativeSearchByPaging(configuration.getBoolean(PROPERTY_ITERATIVE_SEARCH_BY_PAGING, iterativeSearchByPaging)); setIterativeSearchByPagingBatchSize(configuration.getInt(PROPERTY_ITERATIVE_SEARCH_BY_PAGING_BATCH_SIZE, iterativeSearchByPagingBatchSize)); + + setOrgClosureObjects(configuration.getString(PROPERTY_ORG_CLOSURE_OBJECTS, OrgClosureManager.OrgClosureObjects.FOCUS.toString())); + setOrgClosureStartupAction(configuration.getString(PROPERTY_ORG_CLOSURE_STARTUP_ACTION, OrgClosureManager.StartupAction.REBUILD_IF_NEEDED.toString())); + setStopOnOrgClosureStartupFailure(configuration.getBoolean(PROPERTY_STOP_ON_ORG_CLOSURE_STARTUP_FAILURE, true)); } private void computeDefaultConcurrencyParameters() { @@ -472,6 +485,22 @@ public void setUseZip(boolean useZip) { this.useZip = useZip; } + public OrgClosureManager.OrgClosureObjects getOrgClosureObjects() { + return orgClosureObjects; + } + + public void setOrgClosureObjects(String orgClosureObjects) { + this.orgClosureObjects = OrgClosureManager.OrgClosureObjects.fromValue(orgClosureObjects); + } + + public OrgClosureManager.StartupAction getOrgClosureStartupAction() { + return orgClosureStartupAction; + } + + public void setOrgClosureStartupAction(String orgClosureStartupAction) { + this.orgClosureStartupAction = OrgClosureManager.StartupAction.fromValue(orgClosureStartupAction); + } + public boolean isUsingH2() { if (hibernateDialect == null) { return true; @@ -504,4 +533,12 @@ private boolean isUsingDialect(Class dialect) { return false; } + + public void setStopOnOrgClosureStartupFailure(boolean stopOnOrgClosureStartupFailure) { + this.stopOnOrgClosureStartupFailure = stopOnOrgClosureStartupFailure; + } + + public boolean isStopOnOrgClosureStartupFailure() { + return stopOnOrgClosureStartupFailure; + } } diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java index c6b283eabae..470ed6e8b7c 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java @@ -35,13 +35,11 @@ import com.evolveum.midpoint.repo.sql.query.QueryEngine; import com.evolveum.midpoint.repo.sql.query.QueryException; import com.evolveum.midpoint.repo.sql.query.RQuery; -import com.evolveum.midpoint.repo.sql.type.XMLGregorianCalendarType; import com.evolveum.midpoint.repo.sql.util.*; import com.evolveum.midpoint.schema.*; import com.evolveum.midpoint.schema.result.OperationResult; import com.evolveum.midpoint.schema.result.OperationResultStatus; import com.evolveum.midpoint.util.DebugUtil; -import com.evolveum.midpoint.util.QNameUtil; import com.evolveum.midpoint.util.exception.ObjectAlreadyExistsException; import com.evolveum.midpoint.util.exception.ObjectNotFoundException; import com.evolveum.midpoint.util.exception.SchemaException; @@ -51,18 +49,16 @@ import com.evolveum.midpoint.xml.ns._public.common.common_3.*; import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; -import org.apache.commons.lang.NotImplementedException; -import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.Validate; import org.hibernate.*; -import org.hibernate.criterion.Projections; import org.hibernate.criterion.Restrictions; import org.hibernate.exception.ConstraintViolationException; import org.hibernate.internal.SessionFactoryImpl; import org.hibernate.jdbc.Work; import org.springframework.stereotype.Repository; +import javax.annotation.PostConstruct; import javax.xml.namespace.QName; import java.lang.reflect.Method; @@ -539,16 +535,18 @@ private String overwriteAddObjectAttempt(PrismObject o updateFullObject(rObject, object); RObject merged = (RObject) session.merge(rObject); //add and maybe modify - OrgClosureManager.Operation operation; - if (modifications == null) { - operation = OrgClosureManager.Operation.ADD; - modifications = createAddParentRefDelta(object); - } else { - operation = OrgClosureManager.Operation.MODIFY; - } - getClosureManager().updateOrgClosure(oldObject, modifications, session, merged.getOid(), object.getCompileTimeClass(), - operation); + if (getClosureManager().isEnabled()) { + OrgClosureManager.Operation operation; + if (modifications == null) { + operation = OrgClosureManager.Operation.ADD; + modifications = createAddParentRefDelta(object); + } else { + operation = OrgClosureManager.Operation.MODIFY; + } + getClosureManager().updateOrgClosure(oldObject, modifications, session, merged.getOid(), object.getCompileTimeClass(), + operation); + } return merged.getOid(); } @@ -596,9 +594,11 @@ private String nonOverwriteAddObjectAttempt(PrismObject modifications = createAddParentRefDelta(object); - getClosureManager().updateOrgClosure(null, modifications, session, oid, object.getCompileTimeClass(), - OrgClosureManager.Operation.ADD); + if (getClosureManager().isEnabled()) { + Collection modifications = createAddParentRefDelta(object); + getClosureManager().updateOrgClosure(null, modifications, session, oid, object.getCompileTimeClass(), + OrgClosureManager.Operation.ADD); + } return oid; } @@ -996,7 +996,10 @@ private void modifyObjectAttempt(Class type, String oi if (LOGGER.isTraceEnabled()) { LOGGER.trace("OBJECT before:\n{}", new Object[]{prismObject.debugDump()}); } - PrismObject originalObject = prismObject.clone(); + PrismObject originalObject = null; + if (getClosureManager().isEnabled()) { + originalObject = prismObject.clone(); + } ItemDelta.applyTo(modifications, prismObject); if (LOGGER.isTraceEnabled()) { LOGGER.trace("OBJECT after:\n{}", prismObject.debugDump()); @@ -1009,7 +1012,9 @@ private void modifyObjectAttempt(Class type, String oi updateFullObject(rObject, prismObject); session.merge(rObject); - getClosureManager().updateOrgClosure(originalObject, modifications, session, oid, type, OrgClosureManager.Operation.MODIFY); + if (getClosureManager().isEnabled()) { + getClosureManager().updateOrgClosure(originalObject, modifications, session, oid, type, OrgClosureManager.Operation.MODIFY); + } LOGGER.trace("Before commit..."); session.getTransaction().commit(); @@ -1273,6 +1278,11 @@ public void repositorySelfTest(OperationResult parentResult) { // No self-tests for now } + @PostConstruct + public void initialize() { + getClosureManager().executeStartupAction(this); + } + /* (non-Javadoc) * @see com.evolveum.midpoint.repo.api.RepositoryService#getVersion(java.lang.Class, java.lang.String, * com.evolveum.midpoint.schema.result.OperationResult)