Skip to content

Commit

Permalink
Attempt at integrating closure with the rest of midPoint.
Browse files Browse the repository at this point in the history
  • Loading branch information
@mederly
mederly committed Oct 7, 2014
1 parent ed8360c commit 3066dd9
Show file tree
Hide file tree
Showing 8 changed files with 377 additions and 83 deletions.
4 changes: 2 additions & 2 deletions repo/repo-cache/src/main/java/com/evolveum/midpoint/repo/cache/RepositoryCache.java
View file
Expand Up @@ -298,8 +298,8 @@ public RepositoryDiag getRepositoryDiag() {
public void repositorySelfTest(OperationResult parentResult) {
repository.repositorySelfTest(parentResult);
}
private <T extends ObjectType> void cacheObject(Map<String, PrismObject<ObjectType>> cache, PrismObject<T> object) {

private <T extends ObjectType> void cacheObject(Map<String, PrismObject<ObjectType>> cache, PrismObject<T> object) {
if (cache != null) {
cache.put(object.getOid(), (PrismObject<ObjectType>) object.clone());
}
Expand Down
17 changes: 16 additions & 1 deletion ...mpl-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/AbstractOrgClosureTest.java
View file
Expand Up @@ -150,6 +150,8 @@ private void checkChildrenSets(Set<String> oidsToCheck) {
/**
* Recomputes closure table from scratch (using matrix multiplication) and compares it with M_ORG_CLOSURE.
*/
private static final boolean DUMP_TC_MATRIX_DETAILS = false;

protected void checkClosureMatrix() {
Session session = getSession();
// we compute the closure table "by hand" as 1 + A + A^2 + A^3 + ... + A^n where n is the greatest expected path length
Expand All @@ -159,6 +161,9 @@ protected void checkClosureMatrix() {

// used to give indices to vertices
List<String> vertexList = new ArrayList<>(getVertices());

if (DUMP_TC_MATRIX_DETAILS) LOGGER.info("Vertex list = {}", vertexList);

DoubleMatrix2D a = new SparseDoubleMatrix2D(vertices, vertices);
// for (int i = 0; i < vertices; i++) {
// a.setQuick(i, i, 1.0);
Expand All @@ -184,6 +189,8 @@ protected void checkClosureMatrix() {
}
LOGGER.info("TC matrix computed in {} ms", System.currentTimeMillis() - start);

if (DUMP_TC_MATRIX_DETAILS) LOGGER.info("TC matrix expected = {}", result);

Query q = session.createSQLQuery("select descendant_oid, ancestor_oid, val from m_org_closure")
.addScalar("descendant_oid", StringType.INSTANCE)
.addScalar("ancestor_oid", StringType.INSTANCE)
Expand All @@ -193,17 +200,25 @@ protected void checkClosureMatrix() {

DoubleMatrix2D closureInDatabase = new SparseDoubleMatrix2D(vertices, vertices);
for (Object[] item : list) {
int val = Integer.parseInt(item[2].toString());
if (val == 0) {
throw new IllegalStateException("Row with val == 0 in closure table: " + list);
}
closureInDatabase.set(vertexList.indexOf(item[0]),
vertexList.indexOf(item[1]),
Integer.parseInt(item[2].toString()));
val);
}

if (DUMP_TC_MATRIX_DETAILS) LOGGER.info("TC matrix fetched from db = {}", closureInDatabase);

double zSumResultBefore = result.zSum();
double zSumClosureInDb = closureInDatabase.zSum();
result.assign(closureInDatabase, Functions.minus);
double zSumResultAfter = result.zSum();
LOGGER.info("Summary of items in closure computed: {}, in DB-stored closure: {}, delta: {}", new Object[]{zSumResultBefore, zSumClosureInDb, zSumResultAfter});

if (DUMP_TC_MATRIX_DETAILS) LOGGER.info("Difference matrix = {}", result);

boolean problem = false;
for (int i = 0; i < vertices; i++) {
for (int j = 0; j < vertices; j++) {
Expand Down
119 changes: 70 additions & 49 deletions ...-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureConcurrencyTest.java
View file
Expand Up @@ -27,6 +27,7 @@
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.OrgType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.UserType;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.testng.annotations.Test;
Expand Down Expand Up @@ -80,8 +81,8 @@ public OrgClosureTestConfiguration getConfiguration() {
@Test(enabled = true) public void test150CheckClosure() throws Exception { _test150CheckClosure(); }
@Test(enabled = true) public void test200AddRemoveLinksSeq() throws Exception { _test200AddRemoveLinksMT(false); }
@Test(enabled = true) public void test201AddRemoveLinksRandom() throws Exception { _test200AddRemoveLinksMT(true); }
@Test(enabled = true) public void test300AddRemoveOrgsSeq() throws Exception { _test300AddRemoveOrgsMT(false); }
@Test(enabled = true) public void test301AddRemoveOrgsRandom() throws Exception { _test300AddRemoveOrgsMT(true); }
@Test(enabled = true) public void test300AddRemoveNodesSeq() throws Exception { _test300AddRemoveNodesMT(false); }
@Test(enabled = true) public void test301AddRemoveNodesRandom() throws Exception { _test300AddRemoveNodesMT(true); }
//@Test(enabled = true) public void test310AddRemoveUsers() throws Exception { _test310AddRemoveUsersMT(); }

/**
Expand Down Expand Up @@ -154,7 +155,9 @@ public void run() {
LOGGER.info("Removing {}", edge);
removeEdge(edge);
info(Thread.currentThread().getName() + " removed " + edge);
edgesToRemove.remove(edge);
synchronized (OrgClosureConcurrencyTest.this) {
edgesToRemove.remove(edge);
}
}
} catch (Throwable e) {
e.printStackTrace();
Expand Down Expand Up @@ -197,7 +200,9 @@ public void run() {
LOGGER.info("Adding {}", edge);
addEdge(edge);
info(Thread.currentThread().getName() + " re-added " + edge);
edgesToAdd.remove(edge);
synchronized (OrgClosureConcurrencyTest.this) {
edgesToAdd.remove(edge);
}
}
} catch (Throwable e) {
e.printStackTrace();
Expand Down Expand Up @@ -274,41 +279,25 @@ private void addEdge(Edge edge) throws ObjectAlreadyExistsException, ObjectNotFo
}
}

protected void _test300AddRemoveOrgsMT(final boolean random) throws Exception {
OperationResult opResult = new OperationResult("===[ test300AddRemoveOrgsMT ]===");
protected void _test300AddRemoveNodesMT(final boolean random) throws Exception {
OperationResult opResult = new OperationResult("===[ test300AddRemoveNodesMT ]===");

info("test300AddRemoveOrgs starting with random = " + random);
info("test300AddRemoveNodes starting with random = " + random);

final Set<OrgType> orgsToRemove = Collections.synchronizedSet(new HashSet<OrgType>());
final Set<OrgType> orgsToAdd = Collections.synchronizedSet(new HashSet<OrgType>());
final Set<ObjectType> nodesToRemove = Collections.synchronizedSet(new HashSet<ObjectType>());
final Set<ObjectType> nodesToAdd = Collections.synchronizedSet(new HashSet<ObjectType>());

final List<Throwable> exceptions = Collections.synchronizedList(new ArrayList<Throwable>());

for (int level = 0; level < getConfiguration().getNodeRoundsForLevel().length; level++) {
int rounds = getConfiguration().getNodeRoundsForLevel()[level];
List<String> levelOids = orgsByLevels.get(level);
if (levelOids.isEmpty()) {
continue;
}
int retries = 0;
for (int round = 0; round < rounds; round++) {

int index = (int) Math.floor(Math.random() * levelOids.size());
String oid = levelOids.get(index);
OrgType org = repositoryService.getObject(OrgType.class, oid, null, opResult).asObjectable();

if (orgsToRemove.contains(org)) {
round--;
if (++retries == 1000) {
throw new IllegalStateException("Too many retries"); // primitive attempt to break potential cycles when there is not enough edges to process
} else {
continue;
}
}

orgsToRemove.add(org);
orgsToAdd.add(org);
}
generateNodesAtOneLevel(nodesToRemove, nodesToAdd, OrgType.class, rounds, levelOids, opResult);
}
for (int level = 0; level < getConfiguration().getUserRoundsForLevel().length; level++) {
int rounds = getConfiguration().getUserRoundsForLevel()[level];
List<String> levelOids = usersByLevels.get(level);
generateNodesAtOneLevel(nodesToRemove, nodesToAdd, UserType.class, rounds, levelOids, opResult);
}

int numberOfRunners = 3;
Expand All @@ -320,18 +309,20 @@ protected void _test300AddRemoveOrgsMT(final boolean random) throws Exception {
public void run() {
try {
while (true) {
OrgType org = getNext(orgsToRemove, random);
if (org == null) {
ObjectType objectType = getNext(nodesToRemove, random);
if (objectType == null) {
break;
}
LOGGER.info("Removing {}", org);
LOGGER.info("Removing {}", objectType);
try {
removeObject(org);
orgsToRemove.remove(org);
info(Thread.currentThread().getName() + " removed " + org);
removeObject(objectType);
synchronized (OrgClosureConcurrencyTest.this) {
nodesToRemove.remove(objectType);
}
info(Thread.currentThread().getName() + " removed " + objectType);
} catch (ObjectNotFoundException e) {
// this is OK
info(Thread.currentThread().getName() + ": " + org + " already deleted");
info(Thread.currentThread().getName() + ": " + objectType + " already deleted");
}
}
} catch (Throwable e) {
Expand All @@ -351,8 +342,8 @@ public void run() {
Thread.sleep(100); // primitive way of waiting
}

if (!orgsToRemove.isEmpty()) {
throw new AssertionError("Orgs to remove is not empty, see the console or log: " + orgsToRemove);
if (!nodesToRemove.isEmpty()) {
throw new AssertionError("Nodes to remove is not empty, see the console or log: " + nodesToRemove);
}

if (!exceptions.isEmpty()) {
Expand All @@ -368,18 +359,20 @@ public void run() {
public void run() {
try {
while (true) {
OrgType org = getNext(orgsToAdd, random);
if (org == null) {
ObjectType objectType = getNext(nodesToAdd, random);
if (objectType == null) {
break;
}
LOGGER.info("Adding {}", org);
LOGGER.info("Adding {}", objectType);
try {
addObject(org.clone());
orgsToAdd.remove(org);
info(Thread.currentThread().getName() + " re-added " + org);
addObject(objectType.clone());
synchronized (OrgClosureConcurrencyTest.this) {
nodesToAdd.remove(objectType);
}
info(Thread.currentThread().getName() + " re-added " + objectType);
} catch (ObjectAlreadyExistsException e) {
// this is OK
info(Thread.currentThread().getName() + ": " + org + " already exists");
info(Thread.currentThread().getName() + ": " + objectType + " already exists");
}
}
} catch (Throwable e) {
Expand All @@ -399,8 +392,8 @@ public void run() {
Thread.sleep(100); // primitive way of waiting
}

if (!orgsToAdd.isEmpty()) {
throw new AssertionError("Orgs to add is not empty, see the console or log: " + orgsToAdd);
if (!nodesToAdd.isEmpty()) {
throw new AssertionError("Nodes to add is not empty, see the console or log: " + nodesToAdd);
}

if (!exceptions.isEmpty()) {
Expand All @@ -411,6 +404,34 @@ public void run() {
info("Consistency after re-adding OK");
}

private void generateNodesAtOneLevel(Set<ObjectType> nodesToRemove, Set<ObjectType> nodesToAdd,
Class<? extends ObjectType> clazz,
int rounds, List<String> candidateOids,
OperationResult opResult) throws ObjectNotFoundException, SchemaException {
if (candidateOids.isEmpty()) {
return;
}
int retries = 0;
for (int round = 0; round < rounds; round++) {

int index = (int) Math.floor(Math.random() * candidateOids.size());
String oid = candidateOids.get(index);
ObjectType objectType = repositoryService.getObject(clazz, oid, null, opResult).asObjectable();

if (nodesToRemove.contains(objectType)) {
round--;
if (++retries == 1000) {
throw new IllegalStateException("Too many retries"); // primitive attempt to break potential cycles when there is not enough edges to process
} else {
continue;
}
}

nodesToRemove.add(objectType);
nodesToAdd.add(objectType);
}
}

void removeObject(ObjectType objectType) throws Exception {
repositoryService.deleteObject(objectType.getClass(), objectType.getOid(), new OperationResult("dummy"));
synchronized(orgGraph) {
Expand Down
10 changes: 9 additions & 1 deletion ...-test/src/test/java/com/evolveum/midpoint/repo/sql/closure/OrgClosureCorrectnessTest.java
View file
Expand Up @@ -27,13 +27,21 @@
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
public class OrgClosureCorrectnessTest extends AbstractOrgClosureTest {

private static final int[] ORG_CHILDREN_IN_LEVEL = { 3, 2, 3, 0 };
private static final int[] ORG_CHILDREN_IN_LEVEL = { 1, 2, 3, 0 };
private static final int[] USER_CHILDREN_IN_LEVEL = { 0, 1, 2, 3 };
private static final int[] PARENTS_IN_LEVEL = { 0, 2, 2, 3 };
private static final int[] LINK_ROUNDS_FOR_LEVELS = { 0, 5, 10 };
private static final int[] NODE_ROUNDS_FOR_LEVELS = { 1, 5, 10 };
private static final int[] USER_ROUNDS_FOR_LEVELS = { 0, 5 ,5, 10 };

// trivial case for debugging
// private static final int[] ORG_CHILDREN_IN_LEVEL = { 1, 2, 0 };
// private static final int[] USER_CHILDREN_IN_LEVEL = { 0, 0, 0 };
// private static final int[] PARENTS_IN_LEVEL = { 0, 1, 1 };
// private static final int[] LINK_ROUNDS_FOR_LEVELS = { 0, 1, 0 };
// private static final int[] NODE_ROUNDS_FOR_LEVELS = { 1, 5, 10 };
// private static final int[] USER_ROUNDS_FOR_LEVELS = { 0, 5 ,5, 10 };

private OrgClosureTestConfiguration configuration;

public OrgClosureCorrectnessTest() {
Expand Down

0 comments on commit 3066dd9

Please sign in to comment.