diff --git a/icf-connectors/dummy-resource/src/main/java/com/evolveum/icf/dummy/resource/DummyResource.java b/icf-connectors/dummy-resource/src/main/java/com/evolveum/icf/dummy/resource/DummyResource.java index 95e37009c20..421b93d616e 100644 --- a/icf-connectors/dummy-resource/src/main/java/com/evolveum/icf/dummy/resource/DummyResource.java +++ b/icf-connectors/dummy-resource/src/main/java/com/evolveum/icf/dummy/resource/DummyResource.java @@ -97,6 +97,7 @@ public class DummyResource implements DebugDumpable { private boolean caseIgnoreId = false; private boolean caseIgnoreValues = false; private int connectionCount = 0; + private int writeOperationCount = 0; private int groupMembersReadCount = 0; private Collection forbiddenNames; @@ -376,6 +377,14 @@ public synchronized void disconnect() { public void assertNoConnections() { assert connectionCount == 0 : "Dummy resource: "+connectionCount+" connections still open"; } + + public synchronized void recordWriteOperation(String operation) { + writeOperationCount++; + } + + public int getWriteOperationCount() { + return writeOperationCount; + } public int getGroupMembersReadCount() { return groupMembersReadCount; @@ -535,6 +544,7 @@ public Collection listOrgs() throws ConnectException, FileNotFoundExce private synchronized String addObject(Map map, T newObject) throws ObjectAlreadyExistsException, ConnectException, FileNotFoundException, SchemaViolationException, ConflictException { checkBlockOperations(); + recordWriteOperation("add"); breakIt(addBreakMode, "add"); Class type = newObject.getClass(); @@ -583,6 +593,7 @@ private synchronized String addObject(Map map, private synchronized void deleteObjectByName(Class type, Map map, String name) throws ObjectDoesNotExistException, ConnectException, FileNotFoundException, SchemaViolationException, ConflictException { checkBlockOperations(); + recordWriteOperation("delete"); breakIt(deleteBreakMode, "delete"); String normalName = normalize(name); @@ -625,6 +636,7 @@ public void deleteOrgById(String id) throws ConnectException, FileNotFoundExcept private synchronized void deleteObjectById(Class type, Map map, String id) throws ObjectDoesNotExistException, ConnectException, FileNotFoundException, SchemaViolationException, ConflictException { checkBlockOperations(); + recordWriteOperation("delete"); breakIt(deleteBreakMode, "delete"); DummyObject object = allObjects.get(id); @@ -661,6 +673,7 @@ private synchronized void deleteObjectById(Class type private void renameObject(Class type, Map map, String id, String oldName, String newName) throws ObjectDoesNotExistException, ObjectAlreadyExistsException, ConnectException, FileNotFoundException, SchemaViolationException, ConflictException { checkBlockOperations(); + recordWriteOperation("modify"); breakIt(modifyBreakMode, "modify"); T existingObject; @@ -753,6 +766,7 @@ public void renameOrg(String id, String oldName, String newName) throws ObjectDo } void recordModify(DummyObject dObject) { + recordWriteOperation("modify"); if (syncStyle != DummySyncStyle.NONE) { int syncToken = nextSyncToken(); DummyDelta delta = new DummyDelta(syncToken, dObject.getClass(), dObject.getId(), dObject.getName(), DummyDeltaType.MODIFY); diff --git a/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd b/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd index 1861004b4ff..c586348fe07 100644 --- a/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd +++ b/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd @@ -4551,6 +4551,40 @@ in combination with shadowConstraintsCheck to reduce probability of identifier conflicts for resources with slow create/rename operations. + See also avoidDuplicateOperations and recordPendingOperations properties. + + This feature is EXPERIMENTAL. Use with care. + + + 3.6.1 + true + + + + + + + TODO + + See also avoidDuplicateOperations and useProposedShadows properties. + + This feature is EXPERIMENTAL. Use with care. + + + 3.6.1 + true + + + + + + + When set to true, midPoint will try to avoid executing duplication operations + on the resource. If an operation is already underway the duplicate operation + will be ignored. + + See also recordPendingOperations and useProposedShadows properties. + This feature is EXPERIMENTAL. Use with care. @@ -4665,6 +4699,45 @@ + + + + TODO + + + + 3.6.1 + true + + + + + + + Record only asynchronous operations. The operation + will be recorded when we know that it is asynchronous. + Which usually means it is only recoded after the operation + is started. + + + + + + + + + + Record all operations to pending deltas. The operations + are recoreded even before they are started. + + + + + + + + + diff --git a/infra/test-util/src/main/java/com/evolveum/midpoint/test/util/Counter.java b/infra/test-util/src/main/java/com/evolveum/midpoint/test/util/Counter.java new file mode 100644 index 00000000000..75bc1994c73 --- /dev/null +++ b/infra/test-util/src/main/java/com/evolveum/midpoint/test/util/Counter.java @@ -0,0 +1,73 @@ +/** + * Copyright (c) 2017 Evolveum + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.evolveum.midpoint.test.util; + +/** + * @author semancik + * + */ +public class Counter { + private int count = 0; + + public int getCount() { + return count; + } + + public synchronized void click() { + count++; + } + + public void assertCount(int expectedCount) { + assert count == expectedCount : "Wrong counter, expected "+expectedCount+", was "+count; + } + + public void assertCount(String message, int expectedCount) { + assert count == expectedCount : message + ", expected "+expectedCount+", was "+count; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + count; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Counter other = (Counter) obj; + if (count != other.count) { + return false; + } + return true; + } + + @Override + public String toString() { + return "Counter(" + count + ")"; + } + + +} diff --git a/infra/test-util/src/main/java/com/evolveum/midpoint/test/util/ParallelTestThread.java b/infra/test-util/src/main/java/com/evolveum/midpoint/test/util/ParallelTestThread.java new file mode 100644 index 00000000000..50995d3a9b4 --- /dev/null +++ b/infra/test-util/src/main/java/com/evolveum/midpoint/test/util/ParallelTestThread.java @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2016 Evolveum + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.evolveum.midpoint.test.util; + +import com.evolveum.midpoint.util.FailableRunnable; +import com.evolveum.midpoint.util.exception.SystemException; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; + +/** + * @author semancik + * + */ +public class ParallelTestThread extends Thread { + + private static final Trace LOGGER = TraceManager.getTrace(ParallelTestThread.class); + + private FailableRunnable target; + private Throwable exception; + + public ParallelTestThread(FailableRunnable target) { + super(); + this.target = target; + } + + @Override + public void run() { + try { + target.run(); + } catch (RuntimeException | Error e) { + recordException(e); + throw e; + } catch (Throwable e) { + recordException(e); + throw new SystemException(e.getMessage(), e); + } + } + + public Throwable getException() { + return exception; + } + + public void recordException(Throwable e) { + LOGGER.error("Test thread failed: {}", e.getMessage(), e); + this.exception = e; + } + +} diff --git a/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractManualResourceTest.java b/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractManualResourceTest.java index 22619daf375..e6dd0529b9a 100644 --- a/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractManualResourceTest.java +++ b/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractManualResourceTest.java @@ -36,6 +36,7 @@ import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import com.evolveum.midpoint.util.FailableRunnable; import com.evolveum.midpoint.util.exception.*; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext.ClassMode; @@ -75,6 +76,7 @@ import com.evolveum.midpoint.schema.util.ResourceTypeUtil; import com.evolveum.midpoint.task.api.Task; import com.evolveum.midpoint.test.IntegrationTestTools; +import com.evolveum.midpoint.test.util.ParallelTestThread; import com.evolveum.midpoint.test.util.TestUtil; import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; @@ -186,9 +188,6 @@ public abstract class AbstractManualResourceTest extends AbstractConfiguredModel protected static final String INTEREST_ONE = "one"; - protected static final Random RND = new Random(); - - protected PrismObject resource; protected ResourceType resourceType; @@ -2912,42 +2911,29 @@ public void test900ConcurrentConstructions() throws Exception { Task task = createTask(TEST_NAME); OperationResult result = task.getResult(); - final int THREADS = getConcurrentTestNumberOfThreads(); final long TIMEOUT = 60000L; // WHEN displayWhen(TEST_NAME); - Thread[] threads = new Thread[THREADS]; - for (int i = 0; i < THREADS; i++) { - threads[i] = new Thread(() -> { - try { - Thread.sleep(RND.nextInt(getConcurrentTestRandomStartDelayRange())); // Random start delay - LOGGER.info("{} starting", Thread.currentThread().getName()); + + ParallelTestThread[] threads = multithread(TEST_NAME, + () -> { login(userAdministrator); Task localTask = createTask(TEST_NAME + ".local"); + assignAccount(USER_BARBOSSA_OID, getResourceOid(), SchemaConstants.INTENT_DEFAULT, localTask, localTask.getResult()); - } catch (CommonException | InterruptedException e) { - throw new SystemException("Couldn't assign resource: " + e.getMessage(), e); - } - }); - threads[i].setName("Thread " + (i+1) + " of " + THREADS); - threads[i].start(); - } - + + }, getConcurrentTestNumberOfThreads(), getConcurrentTestRandomStartDelayRange()); + // THEN displayThen(TEST_NAME); - for (int i = 0; i < THREADS; i++) { - if (threads[i].isAlive()) { - System.out.println("Waiting for " + threads[i]); - threads[i].join(TIMEOUT); - } - } + waitForThreads(threads, TIMEOUT); PrismObject barbossa = getUser(USER_BARBOSSA_OID); display("barbossa", barbossa); assertEquals("Wrong # of links", 1, barbossa.asObjectable().getLinkRef().size()); } - + protected int getConcurrentTestNumberOfThreads() { return 4; } diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java index 43e107b91c6..bfb68709660 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java @@ -583,6 +583,10 @@ public String addShadow(PrismObject shadow, OperationProvisioningScr isDoDiscovery(resource, options), true, parentResult); return null; } + if (!(attributesContainer instanceof ResourceAttributeContainer)) { + applyAttributesDefinition(ctx, shadow); + attributesContainer = shadow.findContainer(ShadowType.F_ATTRIBUTES); + } preAddChecks(ctx, shadow, task, parentResult); @@ -614,7 +618,7 @@ public String addShadow(PrismObject shadow, OperationProvisioningScr } // REPO OPERATION: add - // This is where the repo shadow is created (if needed) + // This is where the repo shadow is created or updated (if needed) String oid = afterAddOnResource(ctx, shadowOid, asyncReturnValue, parentResult); addedShadow.setOid(oid); diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java index 9dcd4f4d66d..4c8275e0919 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java @@ -846,8 +846,10 @@ public String addNewActiveRepositoryShadow(ProvisioningContext ctx, String exist @SuppressWarnings("unchecked") private void updateProposedShadowAfterAdd(ProvisioningContext ctx, String existingShadowOid, AsynchronousOperationReturnValue> addResult, OperationResult parentResult) throws SchemaException, ConfigurationException, ObjectNotFoundException, CommunicationException, ObjectAlreadyExistsException, ExpressionEvaluationException { PrismObject resourceShadow = addResult.getReturnValue(); + LOGGER.info("AAAAA resource:\n{}", resourceShadow.debugDumpLazily(1)); PrismObject proposedShadow = repositoryService.getObject(ShadowType.class, existingShadowOid, null, parentResult); + LOGGER.info("AAAAA repo:\n{}", proposedShadow.debugDumpLazily(1)); if (proposedShadow == null) { parentResult @@ -856,10 +858,6 @@ private void updateProposedShadowAfterAdd(ProvisioningContext ctx, String existi "Error while creating account shadow object to save in the reposiotory. Draft shadow is gone."); } - // TODO: do better (full) update. There may be UID that came from the resource and so on. - // for now we are just updating lifecycle state and pending operation. This is good for manual connectors. - // But not for regular use. - Collection shadowChanges = new ArrayList<>(); for (PendingOperationType pendingOperation: proposedShadow.asObjectable().getPendingOperation()) { // Remove old ADD pending delta. We'll replace it with newer version below ... if needed. @@ -886,6 +884,8 @@ private void updateProposedShadowAfterAdd(ProvisioningContext ctx, String existi lifecycleDelta.setValuesToReplace(new PrismPropertyValue<>(SchemaConstants.LIFECYCLE_ACTIVE)); shadowChanges.add(lifecycleDelta); + computeUpdateShadowAttributeChanges(ctx, shadowChanges, resourceShadow, proposedShadow); + if (LOGGER.isTraceEnabled()) { LOGGER.trace("Updading draft repository shadow\n{}", DebugUtil.debugDump(shadowChanges, 1)); } @@ -1154,16 +1154,33 @@ private Collection extractRepoShadowChanges(ProvisioningCon public Collection updateShadow(ProvisioningContext ctx, PrismObject resourceShadow, Collection aprioriDeltas, OperationResult result) throws SchemaException, ConfigurationException, ObjectNotFoundException, CommunicationException, ExpressionEvaluationException { PrismObject repoShadow = repositoryService.getObject(ShadowType.class, resourceShadow.getOid(), null, result); - RefinedObjectClassDefinition objectClassDefinition = ctx.getObjectClassDefinition(); + Collection repoShadowChanges = new ArrayList(); - CachingStategyType cachingStrategy = ProvisioningUtil.getCachingStrategy(ctx); + + computeUpdateShadowAttributeChanges(ctx, repoShadowChanges, resourceShadow, repoShadow); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("Updating repo shadow {}:\n{}", resourceShadow.getOid(), DebugUtil.debugDump(repoShadowChanges)); + } + try { + repositoryService.modifyObject(ShadowType.class, resourceShadow.getOid(), repoShadowChanges, result); + } catch (ObjectAlreadyExistsException e) { + // We are not renaming the object here. This should not happen. + throw new SystemException(e.getMessage(), e); + } + return repoShadowChanges; + } + + private void computeUpdateShadowAttributeChanges(ProvisioningContext ctx, Collection repoShadowChanges, + PrismObject resourceShadow, PrismObject repoShadow) throws SchemaException, ConfigurationException, ObjectNotFoundException, CommunicationException, ExpressionEvaluationException { + RefinedObjectClassDefinition objectClassDefinition = ctx.getObjectClassDefinition(); + CachingStategyType cachingStrategy = ProvisioningUtil.getCachingStrategy(ctx); for (RefinedAttributeDefinition attrDef: objectClassDefinition.getAttributeDefinitions()) { if (ProvisioningUtil.shouldStoreAtributeInShadow(objectClassDefinition, attrDef.getName(), cachingStrategy)) { ResourceAttribute resourceAttr = ShadowUtil.getAttribute(resourceShadow, attrDef.getName()); PrismProperty repoAttr = repoShadow.findProperty(new ItemPath(ShadowType.F_ATTRIBUTES, attrDef.getName())); PropertyDelta attrDelta; - if (repoAttr == null && repoAttr == null) { + if (resourceAttr == null && repoAttr == null) { continue; } if (repoAttr == null) { @@ -1181,17 +1198,6 @@ public Collection updateShadow(ProvisioningContext ctx, PrismObject accountRe assertShadowName(accountRepo, ACCOUNT_WILL_USERNAME); assertEquals("Wrong kind (repo)", ShadowKindType.ACCOUNT, accountTypeRepo.getKind()); assertAttribute(accountRepo, SchemaConstants.ICFS_NAME, getWillRepoIcfName()); - if (isIcfNameUidSame()) { + if (isIcfNameUidSame() && !isProposedShadow(accountRepo)) { assertAttribute(accountRepo, SchemaConstants.ICFS_UID, getWillRepoIcfName()); } @@ -1175,6 +1175,14 @@ protected void checkRepoAccountShadowWillBasic(PrismObject accountRe assertRepoCachingMetadata(accountRepo, start, end); } + private boolean isProposedShadow(PrismObject shadow) { + String lifecycleState = shadow.asObjectable().getLifecycleState(); + if (lifecycleState == null) { + return false; + } + return SchemaConstants.LIFECYCLE_PROPOSED.equals(lifecycleState); + } + protected void checkRepoAccountShadowWill(PrismObject accountRepo, XMLGregorianCalendar start, XMLGregorianCalendar end) { checkRepoAccountShadowWillBasic(accountRepo, start, end, 2); assertRepoShadowCacheActivation(accountRepo, null); diff --git a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/dummy/TestDummyParallelism.java b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/dummy/TestDummyParallelism.java new file mode 100644 index 00000000000..bce9a9a6b63 --- /dev/null +++ b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/dummy/TestDummyParallelism.java @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2010-2017 Evolveum + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + */ +package com.evolveum.midpoint.provisioning.impl.dummy; + +import static com.evolveum.midpoint.test.DummyResourceContoller.DUMMY_ACCOUNT_ATTRIBUTE_FULLNAME_NAME; +import static com.evolveum.midpoint.test.DummyResourceContoller.DUMMY_ACCOUNT_ATTRIBUTE_TITLE_NAME; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNull; +import static org.testng.AssertJUnit.assertTrue; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Set; + +import javax.xml.datatype.XMLGregorianCalendar; +import javax.xml.namespace.QName; + +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.testng.AssertJUnit; +import org.testng.annotations.Listeners; +import org.testng.annotations.Test; + +import com.evolveum.icf.dummy.resource.DummyAccount; +import com.evolveum.icf.dummy.resource.DummyGroup; +import com.evolveum.icf.dummy.resource.DummyPrivilege; +import com.evolveum.icf.dummy.resource.DummySyncStyle; +import com.evolveum.midpoint.common.refinery.RefinedResourceSchemaImpl; +import com.evolveum.midpoint.prism.Containerable; +import com.evolveum.midpoint.prism.PrismContainer; +import com.evolveum.midpoint.prism.PrismObject; +import com.evolveum.midpoint.prism.PrismObjectDefinition; +import com.evolveum.midpoint.prism.PrismProperty; +import com.evolveum.midpoint.prism.PrismPropertyValue; +import com.evolveum.midpoint.prism.delta.ChangeType; +import com.evolveum.midpoint.prism.delta.ItemDelta; +import com.evolveum.midpoint.prism.delta.ObjectDelta; +import com.evolveum.midpoint.prism.delta.PropertyDelta; +import com.evolveum.midpoint.prism.path.ItemPath; +import com.evolveum.midpoint.prism.query.AndFilter; +import com.evolveum.midpoint.prism.query.NoneFilter; +import com.evolveum.midpoint.prism.query.ObjectFilter; +import com.evolveum.midpoint.prism.query.ObjectQuery; +import com.evolveum.midpoint.prism.query.builder.QueryBuilder; +import com.evolveum.midpoint.prism.util.PrismAsserts; +import com.evolveum.midpoint.prism.util.PrismTestUtil; +import com.evolveum.midpoint.prism.xml.XmlTypeConverter; +import com.evolveum.midpoint.provisioning.api.ResourceObjectShadowChangeDescription; +import com.evolveum.midpoint.provisioning.impl.ProvisioningTestUtil; +import com.evolveum.midpoint.provisioning.impl.opendj.TestOpenDj; +import com.evolveum.midpoint.schema.DeltaConvertor; +import com.evolveum.midpoint.schema.GetOperationOptions; +import com.evolveum.midpoint.schema.ResourceShadowDiscriminator; +import com.evolveum.midpoint.schema.ResultHandler; +import com.evolveum.midpoint.schema.SearchResultList; +import com.evolveum.midpoint.schema.SelectorOptions; +import com.evolveum.midpoint.schema.constants.SchemaConstants; +import com.evolveum.midpoint.schema.internals.InternalCounters; +import com.evolveum.midpoint.schema.processor.ObjectClassComplexTypeDefinition; +import com.evolveum.midpoint.schema.processor.ResourceAttribute; +import com.evolveum.midpoint.schema.processor.ResourceAttributeContainer; +import com.evolveum.midpoint.schema.processor.ResourceAttributeDefinition; +import com.evolveum.midpoint.schema.processor.ResourceSchema; +import com.evolveum.midpoint.schema.result.OperationResult; +import com.evolveum.midpoint.schema.util.ObjectQueryUtil; +import com.evolveum.midpoint.schema.util.ResourceTypeUtil; +import com.evolveum.midpoint.schema.util.SchemaTestConstants; +import com.evolveum.midpoint.schema.util.ShadowUtil; +import com.evolveum.midpoint.task.api.Task; +import com.evolveum.midpoint.test.DummyResourceContoller; +import com.evolveum.midpoint.test.IntegrationTestTools; +import com.evolveum.midpoint.test.ObjectChecker; +import com.evolveum.midpoint.test.ProvisioningScriptSpec; +import com.evolveum.midpoint.test.util.Counter; +import com.evolveum.midpoint.test.util.ParallelTestThread; +import com.evolveum.midpoint.test.util.TestUtil; +import com.evolveum.midpoint.util.Holder; +import com.evolveum.midpoint.util.exception.CommunicationException; +import com.evolveum.midpoint.util.exception.ConfigurationException; +import com.evolveum.midpoint.util.exception.ExpressionEvaluationException; +import com.evolveum.midpoint.util.exception.ObjectAlreadyExistsException; +import com.evolveum.midpoint.util.exception.ObjectNotFoundException; +import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.util.exception.SecurityViolationException; +import com.evolveum.midpoint.util.exception.SystemException; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.xml.ns._public.common.api_types_3.ObjectModificationType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ActivationStatusType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ActivationType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.LockoutStatusType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationProvisioningScriptsType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultStatusType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ProvisioningScriptType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowKindType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.SynchronizationSituationType; + +/** + * The test of Provisioning service on the API level. + * + * This test is focused on parallelism and race conditions. + * + * The test is using dummy resource for speed and flexibility. + * + * @author Radovan Semancik + * + */ +@ContextConfiguration(locations = "classpath:ctx-provisioning-test-main.xml") +@DirtiesContext +@Listeners({ com.evolveum.midpoint.tools.testng.AlphabeticalMethodInterceptor.class }) +public class TestDummyParallelism extends AbstractBasicDummyTest { + + private static final Trace LOGGER = TraceManager.getTrace(TestDummyParallelism.class); + + public static final File TEST_DIR = new File(TEST_DIR_DUMMY, "dummy-parallelism"); + public static final File RESOURCE_DUMMY_FILE = new File(TEST_DIR, "resource-dummy.xml"); + + private static final long WAIT_TIMEOUT = 60000L; + + private String accountMorganOid; + + protected int getConcurrentTestNumberOfThreads() { + return 5; + } + + protected int getConcurrentTestRandomStartDelayRange() { + return 10; + } + + @Override + public void initSystem(Task initTask, OperationResult initResult) throws Exception { + super.initSystem(initTask, initResult); +// InternalMonitor.setTraceConnectorOperation(true); + } + + @Override + protected File getResourceDummyFilename() { + return RESOURCE_DUMMY_FILE; + } + + // test000-test100 in the superclasses + + @Test + public void test200ParallelCreate() throws Exception { + final String TEST_NAME = "test200ParallelCreate"; + displayTestTitle(TEST_NAME); + // GIVEN + Task task = createTask(TEST_NAME); + OperationResult result = task.getResult(); + + final Counter successCounter = new Counter(); + rememberDummyResourceWriteOperationCount(null); + + // WHEN + displayWhen(TEST_NAME); + + ParallelTestThread[] threads = multithread(TEST_NAME, + () -> { + Task localTask = createTask(TEST_NAME + ".local"); + OperationResult localResult = localTask.getResult(); + + ShadowType account = parseObjectType(ACCOUNT_MORGAN_FILE, ShadowType.class); + + try { + accountMorganOid = provisioningService.addObject(account.asPrismObject(), null, null, localTask, localResult); + successCounter.click(); + } catch (ObjectAlreadyExistsException e) { + // this is expected ... sometimes + LOGGER.info("Exception (maybe expected): {}: {}", e.getClass().getSimpleName(), e.getMessage()); + } + + }, getConcurrentTestNumberOfThreads(), getConcurrentTestRandomStartDelayRange()); + + // THEN + displayThen(TEST_NAME); + waitForThreads(threads, WAIT_TIMEOUT); + + successCounter.assertCount("Wrong number of successful operations", 1); + + PrismObject shadowAfter = provisioningService.getObject(ShadowType.class, accountMorganOid, null, task, result); + display("Shadow after", shadowAfter); + + assertDummyResourceWriteOperationCountIncrement(null, 1); + + assertSteadyResource(); + } + + +} diff --git a/provisioning/provisioning-impl/src/test/resources/dummy/dummy-parallelism/resource-dummy.xml b/provisioning/provisioning-impl/src/test/resources/dummy/dummy-parallelism/resource-dummy.xml new file mode 100644 index 00000000000..1aa52790bbc --- /dev/null +++ b/provisioning/provisioning-impl/src/test/resources/dummy/dummy-parallelism/resource-dummy.xml @@ -0,0 +1,203 @@ + + + + + + Dummy Resource + + + + + + Shiver me timbers! + + Dead men tell no tales + + true + + + + false + false + false + + + + http://midpoint.evolveum.com/xml/ns/public/resource/instance/ef2bc95b-76e0-59e2-86d6-9999dddddddd + + + + account + default + Default Account + true + ri:AccountObjectClass + + icfs:name + Username + + + ri:weapon + mr:stringIgnoreCase + + + ri:loot + explicit + + + ri:ship + + + true + true + true + + + + + ri:drink + + + true + false + true + + + + + ri:quote + + + true + true + false + + + + + ri:gossip + + + false + true + true + + + + + ri:water + + + false + false + false + + + + + ri:group + entitlement + group + objectToSubject + ri:members + icfs:name + + + ri:priv + entitlement + privilege + subjectToObject + ri:privileges + icfs:name + + + root + + + daemon + + + + + + declare namespace icfs="http://midpoint.evolveum.com/xml/ns/public/connector/icf-1/resource-schema-3"; + attributes/icfs:name + + daviejones + + + + + + + + declare namespace icfs="http://midpoint.evolveum.com/xml/ns/public/connector/icf-1/resource-schema-3"; + attributes/icfs:name + + X + true + + + + + + + + declare namespace icfs="http://midpoint.evolveum.com/xml/ns/public/connector/icf-1/resource-schema-3"; + attributes/icfs:name + + -adm + true + + + + + + entitlement + group + true + ri:GroupObjectClass + + ri:members + minimal + + + + entitlement + privilege + false + ri:CustomprivilegeObjectClass + + + + + light + true + + + + diff --git a/provisioning/provisioning-impl/testng-integration.xml b/provisioning/provisioning-impl/testng-integration.xml index 72a4f94974f..7276f526eba 100644 --- a/provisioning/provisioning-impl/testng-integration.xml +++ b/provisioning/provisioning-impl/testng-integration.xml @@ -56,6 +56,7 @@ + diff --git a/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java b/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java index 934218746c3..23e838eb7c6 100644 --- a/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java +++ b/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java @@ -87,10 +87,12 @@ import com.evolveum.midpoint.test.ldap.OpenDJController; import com.evolveum.midpoint.test.util.DerbyController; import com.evolveum.midpoint.test.util.MidPointTestConstants; +import com.evolveum.midpoint.test.util.ParallelTestThread; import com.evolveum.midpoint.test.util.TestUtil; import com.evolveum.midpoint.util.DOMUtil; import com.evolveum.midpoint.util.DebugDumpable; import com.evolveum.midpoint.util.FailableProcessor; +import com.evolveum.midpoint.util.FailableRunnable; import com.evolveum.midpoint.util.LocalizableMessage; import com.evolveum.midpoint.util.MiscUtil; import com.evolveum.midpoint.util.PrettyPrinter; @@ -142,6 +144,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -168,6 +171,8 @@ public abstract class AbstractIntegrationTest extends AbstractTestNGSpringContex private static final Trace LOGGER = TraceManager.getTrace(AbstractIntegrationTest.class); + protected static final Random RND = new Random(); + // Values used to check if something is unchanged or changed properly protected LdapShaPasswordEncoder ldapShaPasswordEncoder = new LdapShaPasswordEncoder(); @@ -181,6 +186,7 @@ public abstract class AbstractIntegrationTest extends AbstractTestNGSpringContex protected RepositoryService repositoryService; protected static Set initializedClasses = new HashSet(); private long lastDummyResourceGroupMembersReadCount; + private long lastDummyResourceWriteOperationCount; @Autowired(required = true) protected TaskManager taskManager; @@ -986,10 +992,21 @@ protected void rememberDummyResourceGroupMembersReadCount(String instanceName) { protected void assertDummyResourceGroupMembersReadCountIncrement(String instanceName, int expectedIncrement) { long currentDummyResourceGroupMembersReadCount = DummyResource.getInstance(instanceName).getGroupMembersReadCount(); long actualIncrement = currentDummyResourceGroupMembersReadCount - lastDummyResourceGroupMembersReadCount; - assertEquals("Unexpected increment in group members read count count in dummy resource '"+instanceName+"'", (long)expectedIncrement, actualIncrement); + assertEquals("Unexpected increment in group members read count in dummy resource '"+instanceName+"'", (long)expectedIncrement, actualIncrement); lastDummyResourceGroupMembersReadCount = currentDummyResourceGroupMembersReadCount; } + protected void rememberDummyResourceWriteOperationCount(String instanceName) { + lastDummyResourceWriteOperationCount = DummyResource.getInstance(instanceName).getWriteOperationCount(); + } + + protected void assertDummyResourceWriteOperationCountIncrement(String instanceName, int expectedIncrement) { + long currentCount = DummyResource.getInstance(instanceName).getWriteOperationCount(); + long actualIncrement = currentCount - lastDummyResourceWriteOperationCount; + assertEquals("Unexpected increment in write operation count in dummy resource '"+instanceName+"'", (long)expectedIncrement, actualIncrement); + lastDummyResourceWriteOperationCount = currentCount; + } + protected PrismObject createShadow(PrismObject resource, String id) throws SchemaException { return createShadow(resource, id, id); } @@ -2065,4 +2082,32 @@ protected void assertExceptionUserFriendly(CommonException e, String expectedMes assertEquals("Unexpected user friendly exception fallback message", expectedMessage, userFriendlyMessage.getFallbackMessage()); } + protected ParallelTestThread[] multithread(final String TEST_NAME, FailableRunnable lambda, int numberOfThreads, Integer randomStartDelayRange) { + ParallelTestThread[] threads = new ParallelTestThread[numberOfThreads]; + for (int i = 0; i < numberOfThreads; i++) { + threads[i] = new ParallelTestThread(() -> { + if (randomStartDelayRange != null) { + Thread.sleep(RND.nextInt(randomStartDelayRange)); // Random start delay + } + LOGGER.info("{} starting", Thread.currentThread().getName()); + lambda.run(); + }); + threads[i].setName("Thread " + (i+1) + " of " + numberOfThreads); + threads[i].start(); + } + return threads; + } + + protected void waitForThreads(ParallelTestThread[] threads, long timeout) throws InterruptedException { + for (int i = 0; i < threads.length; i++) { + if (threads[i].isAlive()) { + System.out.println("Waiting for " + threads[i]); + threads[i].join(timeout); + Throwable threadException = threads[i].getException(); + if (threadException != null) { + throw new AssertionError("Test thread "+i+" failed: "+threadException.getMessage(), threadException); + } + } + } + } }