Skip to content

Commit

Permalink
Improved parallel provisioning tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@semancik
semancik committed Sep 12, 2017
1 parent 1aed112 commit 2231fbb
Show file tree
Hide file tree
Showing 2 changed files with 253 additions and 18 deletions.
254 changes: 240 additions & 14 deletions ...mpl/src/test/java/com/evolveum/midpoint/provisioning/impl/dummy/TestDummyParallelism.java
View file
Expand Up @@ -28,6 +28,7 @@

import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.prism.delta.ObjectDelta;
import com.evolveum.midpoint.repo.cache.RepositoryCache;
import com.evolveum.midpoint.schema.constants.SchemaConstants;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.task.api.Task;
Expand Down Expand Up @@ -67,15 +68,20 @@ public class TestDummyParallelism extends AbstractBasicDummyTest {
private static final int DUMMY_OPERATION_DELAY_RANGE = 1000;

private String accountMorganOid;

private String accountElizabethOid;

protected int getConcurrentTestNumberOfThreads() {
return 5;
}
protected int getConcurrentTestRandomStartDelayRange() {

protected int getConcurrentTestFastRandomStartDelayRange() {
return 10;
}


protected int getConcurrentTestSlowRandomStartDelayRange() {
return 1000;
}

@Override
public void initSystem(Task initTask, OperationResult initResult) throws Exception {
super.initSystem(initTask, initResult);
Expand Down Expand Up @@ -118,9 +124,9 @@ public void test200ParallelCreate() throws Exception {
// this is expected ... sometimes
LOGGER.info("Exception (maybe expected): {}: {}", e.getClass().getSimpleName(), e.getMessage());
}
}, getConcurrentTestNumberOfThreads(), getConcurrentTestRandomStartDelayRange());

}, getConcurrentTestNumberOfThreads(), getConcurrentTestFastRandomStartDelayRange());

// THEN
displayThen(TEST_NAME);
waitForThreads(threads, WAIT_TIMEOUT);
Expand Down Expand Up @@ -190,8 +196,13 @@ private PrismObject<ShadowType> parallelModifyTest(final String TEST_NAME, Faila
(i) -> {
Task localTask = createTask(TEST_NAME + ".local");
OperationResult localResult = localTask.getResult();

RepositoryCache.enter();
// Playing with cache, trying to make a worst case
PrismObject<ShadowType> shadowBefore = repositoryService.getObject(ShadowType.class, accountMorganOid, null, localResult);

display("Thread "+Thread.currentThread().getName()+" START");
randomDelay(getConcurrentTestSlowRandomStartDelayRange());
LOGGER.info("{} starting to do some work", Thread.currentThread().getName());

ObjectDelta<ShadowType> delta = deltaProducer.run();
display("ObjectDelta", delta);
Expand All @@ -207,8 +218,10 @@ private PrismObject<ShadowType> parallelModifyTest(final String TEST_NAME, Faila
} else {
fail("Unexpected thread result status " + localResult.getStatus());
}

RepositoryCache.exit();

}, getConcurrentTestNumberOfThreads(), getConcurrentTestRandomStartDelayRange());
}, getConcurrentTestNumberOfThreads(), getConcurrentTestFastRandomStartDelayRange());

// THEN
displayThen(TEST_NAME);
Expand All @@ -227,8 +240,8 @@ private PrismObject<ShadowType> parallelModifyTest(final String TEST_NAME, Faila
}

@Test
public void test229ParallelDelete() throws Exception {
final String TEST_NAME = "test229ParallelDelete";
public void test209ParallelDelete() throws Exception {
final String TEST_NAME = "test209ParallelDelete";
displayTestTitle(TEST_NAME);

// GIVEN
Expand All @@ -243,6 +256,8 @@ public void test229ParallelDelete() throws Exception {
Task localTask = createTask(TEST_NAME + ".local");
OperationResult localResult = localTask.getResult();

RepositoryCache.enter();

try {
display("Thread "+Thread.currentThread().getName()+" START");
provisioningService.deleteObject(ShadowType.class, accountMorganOid, null, null, localTask, localResult);
Expand All @@ -258,10 +273,12 @@ public void test229ParallelDelete() throws Exception {
} catch (ObjectNotFoundException e) {
// this is expected ... sometimes
LOGGER.info("Exception (maybe expected): {}: {}", e.getClass().getSimpleName(), e.getMessage());
} finally {
RepositoryCache.exit();
}
}, getConcurrentTestNumberOfThreads(), getConcurrentTestRandomStartDelayRange());

}, getConcurrentTestNumberOfThreads(), getConcurrentTestFastRandomStartDelayRange());

// THEN
displayThen(TEST_NAME);
waitForThreads(threads, WAIT_TIMEOUT);
Expand All @@ -275,4 +292,213 @@ public void test229ParallelDelete() throws Exception {
assertSteadyResource();
}

@Test
public void test210ParallelCreateSlow() throws Exception {
final String TEST_NAME = "test210ParallelCreateSlow";
displayTestTitle(TEST_NAME);
// GIVEN
Task task = createTask(TEST_NAME);
OperationResult result = task.getResult();

final Counter successCounter = new Counter();
rememberDummyResourceWriteOperationCount(null);

// WHEN
displayWhen(TEST_NAME);

ParallelTestThread[] threads = multithread(TEST_NAME,
(i) -> {
Task localTask = createTask(TEST_NAME + ".local");
OperationResult localResult = localTask.getResult();

RepositoryCache.enter();

randomDelay(getConcurrentTestSlowRandomStartDelayRange());
LOGGER.info("{} starting to do some work", Thread.currentThread().getName());

ShadowType account = parseObjectType(ACCOUNT_ELIZABETH_FILE, ShadowType.class);

try {
accountElizabethOid = provisioningService.addObject(account.asPrismObject(), null, null, localTask, localResult);
successCounter.click();
} catch (ObjectAlreadyExistsException e) {
// this is expected ... sometimes
LOGGER.info("Exception (maybe expected): {}: {}", e.getClass().getSimpleName(), e.getMessage());
} finally {
RepositoryCache.exit();
}

}, getConcurrentTestNumberOfThreads(), null);

// THEN
displayThen(TEST_NAME);
waitForThreads(threads, WAIT_TIMEOUT);

successCounter.assertCount("Wrong number of successful operations", 1);

PrismObject<ShadowType> shadowAfter = provisioningService.getObject(ShadowType.class, accountElizabethOid, null, task, result);
display("Shadow after", shadowAfter);

assertDummyResourceWriteOperationCountIncrement(null, 1);

assertSteadyResource();
}

/**
* Create a lot parallel modifications for the same property and the same value.
* These should all be eliminated - except for one of them.
*
* There is a slight chance that one of the thread starts after the first operation
* is finished. But the threads are fast and the operations are slow. So this is
* a very slim chance.
*/
@Test
public void test212ParallelModifyElizabethSlow() throws Exception {
final String TEST_NAME = "test212ParallelModifyElizabethSlow";

PrismObject<ShadowType> shadowAfter = parallelModifyTestSlow(TEST_NAME,
() -> ObjectDelta.createModificationReplaceProperty(ShadowType.class,
accountElizabethOid, dummyResourceCtl.getAttributeFullnamePath(), prismContext, "Miss Swan"));

assertAttribute(shadowAfter, DummyResourceContoller.DUMMY_ACCOUNT_ATTRIBUTE_FULLNAME_NAME, "Miss Swan");
}

/**
* Create a lot parallel modifications for the same property and the same value.
* These should all be eliminated - except for one of them.
*
* There is a slight chance that one of the thread starts after the first operation
* is finished. But the threads are fast and the operations are slow. So this is
* a very slim chance.
*/
@Test
public void test214ParallelModifyDisableSlow() throws Exception {
final String TEST_NAME = "test214ParallelModifyDisableSlow";

PrismObject<ShadowType> shadowAfter = parallelModifyTestSlow(TEST_NAME,
() -> ObjectDelta.createModificationReplaceProperty(ShadowType.class,
accountElizabethOid, SchemaConstants.PATH_ACTIVATION_ADMINISTRATIVE_STATUS, prismContext, ActivationStatusType.DISABLED));

assertActivationAdministrativeStatus(shadowAfter, ActivationStatusType.DISABLED);
}

private PrismObject<ShadowType> parallelModifyTestSlow(final String TEST_NAME, FailableProducer<ObjectDelta<ShadowType>> deltaProducer) throws Exception {
displayTestTitle(TEST_NAME);

// GIVEN
Task task = createTask(TEST_NAME);
OperationResult result = task.getResult();

final Counter successCounter = new Counter();
rememberDummyResourceWriteOperationCount(null);

// WHEN
displayWhen(TEST_NAME);

ParallelTestThread[] threads = multithread(TEST_NAME,
(i) -> {
Task localTask = createTask(TEST_NAME + ".local");
OperationResult localResult = localTask.getResult();

RepositoryCache.enter();
// Playing with cache, trying to make a worst case
PrismObject<ShadowType> shadowBefore = repositoryService.getObject(ShadowType.class, accountElizabethOid, null, localResult);

randomDelay(getConcurrentTestSlowRandomStartDelayRange());
LOGGER.info("{} starting to do some work", Thread.currentThread().getName());

ObjectDelta<ShadowType> delta = deltaProducer.run();
display("ObjectDelta", delta);

provisioningService.modifyObject(ShadowType.class, accountElizabethOid, delta.getModifications(), null, null, localTask, localResult);

localResult.computeStatus();
display("Thread "+Thread.currentThread().getName()+" DONE, result", localResult);
if (localResult.isSuccess()) {
successCounter.click();
} else if (localResult.isInProgress()) {
// expected
} else {
fail("Unexpected thread result status " + localResult.getStatus());
}

RepositoryCache.exit();

}, getConcurrentTestNumberOfThreads(), null);

// THEN
displayThen(TEST_NAME);
waitForThreads(threads, WAIT_TIMEOUT);

PrismObject<ShadowType> shadowAfter = provisioningService.getObject(ShadowType.class, accountElizabethOid, null, task, result);
display("Shadow after", shadowAfter);

successCounter.assertCount("Wrong number of successful operations", 1);

assertDummyResourceWriteOperationCountIncrement(null, 1);

assertSteadyResource();

return shadowAfter;
}


@Test
public void test229ParallelDeleteSlow() throws Exception {
final String TEST_NAME = "test229ParallelDeleteSlow";
displayTestTitle(TEST_NAME);

// GIVEN
final Counter successCounter = new Counter();
rememberDummyResourceWriteOperationCount(null);

// WHEN
displayWhen(TEST_NAME);

ParallelTestThread[] threads = multithread(TEST_NAME,
(i) -> {
Task localTask = createTask(TEST_NAME + ".local");
OperationResult localResult = localTask.getResult();

RepositoryCache.enter();
// Playing with cache, trying to make a worst case
PrismObject<ShadowType> shadowBefore = repositoryService.getObject(ShadowType.class, accountElizabethOid, null, localResult);

randomDelay(getConcurrentTestSlowRandomStartDelayRange());
LOGGER.info("{} starting to do some work", Thread.currentThread().getName());

try {
display("Thread "+Thread.currentThread().getName()+" START");
provisioningService.deleteObject(ShadowType.class, accountElizabethOid, null, null, localTask, localResult);
localResult.computeStatus();
display("Thread "+Thread.currentThread().getName()+" DONE, result", localResult);
if (localResult.isSuccess()) {
successCounter.click();
} else if (localResult.isInProgress()) {
// expected
} else {
fail("Unexpected thread result status " + localResult.getStatus());
}
} catch (ObjectNotFoundException e) {
// this is expected ... sometimes
LOGGER.info("Exception (maybe expected): {}: {}", e.getClass().getSimpleName(), e.getMessage());
} finally {
RepositoryCache.exit();
}

}, getConcurrentTestNumberOfThreads(), null);

// THEN
displayThen(TEST_NAME);
waitForThreads(threads, WAIT_TIMEOUT);

successCounter.assertCount("Wrong number of successful operations", 1);

assertNoRepoObject(ShadowType.class, accountElizabethOid);

assertDummyResourceWriteOperationCountIncrement(null, 1);

assertSteadyResource();
}

}
17 changes: 13 additions & 4 deletions repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java
View file
Expand Up @@ -2087,10 +2087,8 @@ protected ParallelTestThread[] multithread(final String TEST_NAME, MultithreadRu
ParallelTestThread[] threads = new ParallelTestThread[numberOfThreads];
for (int i = 0; i < numberOfThreads; i++) {
threads[i] = new ParallelTestThread(i,
(ii) -> {
if (randomStartDelayRange != null) {
Thread.sleep(RND.nextInt(randomStartDelayRange)); // Random start delay
}
(ii) -> {
randomDelay(randomStartDelayRange);
LOGGER.info("{} starting", Thread.currentThread().getName());
lambda.run(ii);
});
Expand All @@ -2100,6 +2098,17 @@ protected ParallelTestThread[] multithread(final String TEST_NAME, MultithreadRu
return threads;
}

protected void randomDelay(Integer range) {
if (range == null) {
return;
}
try {
Thread.sleep(RND.nextInt(range));
} catch (InterruptedException e) {
// Nothing to do, really
}
}

protected void waitForThreads(ParallelTestThread[] threads, long timeout) throws InterruptedException {
for (int i = 0; i < threads.length; i++) {
if (threads[i].isAlive()) {
Expand Down

0 comments on commit 2231fbb

Please sign in to comment.