From 597adb27a8c49c9b67921bb2334bdcc73ba59171 Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Tue, 8 Oct 2019 13:50:00 +0200 Subject: [PATCH] Refactor ShadowManager a little bit Here we moved ShadowManager to a separate package and factored out delta computation functionality to ShadowDeltaComputer class. Also added tests for faulty update of index-only cached attributes (MID-5832). --- .../src/test/resources/logback-test.xml | 2 +- .../impl/ResourceObjectReferenceResolver.java | 3 +- .../provisioning/impl/ShadowCache.java | 37 ++- .../ObjectAlreadyExistHandler.java | 3 +- .../errorhandling/ObjectNotFoundHandler.java | 2 +- .../shadowmanager/ShadowDeltaComputer.java | 232 ++++++++++++++++++ .../{ => shadowmanager}/ShadowManager.java | 212 ++-------------- .../impl/sync/ChangeProcessor.java | 4 +- .../impl/async/TestAsyncUpdate.java | 162 ++++++++++-- .../impl/async/TestAsyncUpdateCaching.java | 4 +- .../TestAsyncUpdateCachingIndexOnly.java | 5 - .../impl/async/TestAsyncUpdateNoCaching.java | 4 +- ...change-110-banderson-delta-add-values.xml} | 0 ...ange-115-banderson-delta-delete-values.xml | 33 +++ ...nge-117-banderson-delta-replace-values.xml | 33 +++ .../provisioning-impl/testng-integration.xml | 1 + .../test/AbstractIntegrationTest.java | 4 +- .../asserter/ShadowAttributesAsserter.java | 47 +++- 18 files changed, 526 insertions(+), 262 deletions(-) create mode 100644 provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadowmanager/ShadowDeltaComputer.java rename provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/{ => shadowmanager}/ShadowManager.java (90%) rename provisioning/provisioning-impl/src/test/resources/async/{change-110-banderson-delta.xml => change-110-banderson-delta-add-values.xml} (100%) create mode 100644 provisioning/provisioning-impl/src/test/resources/async/change-115-banderson-delta-delete-values.xml create mode 100644 provisioning/provisioning-impl/src/test/resources/async/change-117-banderson-delta-replace-values.xml diff --git a/model/model-intest/src/test/resources/logback-test.xml b/model/model-intest/src/test/resources/logback-test.xml index c09029ec8a3..7bf19e85c7a 100644 --- a/model/model-intest/src/test/resources/logback-test.xml +++ b/model/model-intest/src/test/resources/logback-test.xml @@ -87,7 +87,7 @@ - + diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ResourceObjectReferenceResolver.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ResourceObjectReferenceResolver.java index 679569c3399..17b0a94a6ed 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ResourceObjectReferenceResolver.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ResourceObjectReferenceResolver.java @@ -11,6 +11,7 @@ import javax.xml.namespace.QName; +import com.evolveum.midpoint.provisioning.impl.shadowmanager.ShadowManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; @@ -32,9 +33,7 @@ import com.evolveum.midpoint.repo.common.expression.ExpressionFactory; import com.evolveum.midpoint.repo.common.expression.ExpressionUtil; import com.evolveum.midpoint.repo.common.expression.ExpressionVariables; -import com.evolveum.midpoint.schema.GetOperationOptions; import com.evolveum.midpoint.schema.ResultHandler; -import com.evolveum.midpoint.schema.SelectorOptions; import com.evolveum.midpoint.schema.processor.ResourceAttribute; import com.evolveum.midpoint.schema.processor.ResourceObjectIdentification; import com.evolveum.midpoint.schema.result.OperationResult; diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java index eb4e96dc16f..f0a2ce954b2 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java @@ -23,6 +23,7 @@ import com.evolveum.midpoint.provisioning.api.*; import com.evolveum.midpoint.provisioning.impl.errorhandling.ErrorHandler; import com.evolveum.midpoint.provisioning.impl.errorhandling.ErrorHandlerLocator; +import com.evolveum.midpoint.provisioning.impl.shadowmanager.ShadowManager; import com.evolveum.midpoint.provisioning.ucf.api.*; import com.evolveum.midpoint.provisioning.util.ProvisioningUtil; import com.evolveum.midpoint.repo.api.RepositoryService; @@ -252,7 +253,6 @@ public PrismObject getShadow(String oid, PrismObject rep Collection> identifiers = ShadowUtil.getAllIdentifiers(repositoryShadow); try { - try { resourceShadow = resourceObjectConverter.getResourceObject(ctx, identifiers, true, parentResult); @@ -289,30 +289,23 @@ public PrismObject getShadow(String oid, PrismObject rep resourceShadow.asObjectable().setIntent(repositoryShadow.asObjectable().getIntent()); ProvisioningContext shadowCtx = ctx.spawn(resourceShadow); - resourceManager.modifyResourceAvailabilityStatus(resource.asPrismObject(), - AvailabilityStatusType.UP, parentResult); + resourceManager.modifyResourceAvailabilityStatus(resource.asPrismObject(), AvailabilityStatusType.UP, parentResult); if (LOGGER.isTraceEnabled()) { LOGGER.trace("Shadow from repository:\n{}", repositoryShadow.debugDump(1)); LOGGER.trace("Resource object fetched from resource:\n{}", resourceShadow.debugDump(1)); } - repositoryShadow = shadowManager.updateShadow(shadowCtx, resourceShadow, repositoryShadow, - shadowState, parentResult); - if (LOGGER.isTraceEnabled()) { - LOGGER.trace("Repository shadow after update:\n{}", repositoryShadow.debugDump(1)); - } + repositoryShadow = shadowManager.updateShadow(shadowCtx, resourceShadow, repositoryShadow, null, shadowState, parentResult); + LOGGER.trace("Repository shadow after update:\n{}", repositoryShadow.debugDumpLazily(1)); + // Complete the shadow by adding attributes from the resource object - PrismObject resultShadow = completeShadow(shadowCtx, resourceShadow, repositoryShadow, false, parentResult); + PrismObject assembledShadow = completeShadow(shadowCtx, resourceShadow, repositoryShadow, false, parentResult); + LOGGER.trace("Shadow when assembled:\n{}", assembledShadow.debugDumpLazily(1)); - if (LOGGER.isTraceEnabled()) { - LOGGER.trace("Shadow when assembled:\n{}", resultShadow.debugDump(1)); - } + PrismObject resultShadow = futurizeShadow(ctx, repositoryShadow, assembledShadow, options, now); + LOGGER.trace("Futurized assembled shadow:\n{}", resultShadow.debugDumpLazily(1)); - resultShadow = futurizeShadow(ctx, repositoryShadow, resultShadow, options, now); - if (LOGGER.isTraceEnabled()) { - LOGGER.trace("Futurized assembled shadow:\n{}", resultShadow.debugDump(1)); - } parentResult.recordSuccess(); validateShadow(resultShadow, true); return resultShadow; @@ -329,9 +322,9 @@ public PrismObject getShadow(String oid, PrismObject rep // is returned parentResult.setStatus(OperationResultStatus.PARTIAL_ERROR); } - handledShadow = futurizeShadow(ctx, handledShadow, null, options, now); - validateShadow(handledShadow, true); - return handledShadow; + PrismObject futurizedShadow = futurizeShadow(ctx, handledShadow, null, options, now); + validateShadow(futurizedShadow, true); + return futurizedShadow; } catch (GenericFrameworkException | ObjectAlreadyExistsException | PolicyViolationException e) { throw new SystemException(e.getMessage(), e); @@ -344,7 +337,6 @@ public PrismObject getShadow(String oid, PrismObject rep // fetch for protected objects? if (!ShadowUtil.isProtected(resourceShadow)) { InternalMonitor.recordCount(InternalCounters.SHADOW_FETCH_OPERATION_COUNT); - } } } @@ -1911,9 +1903,12 @@ public SearchResultMetadata searchObjectsIterative(final ProvisioningContext ctx // shadow should have proper kind/intent ProvisioningContext shadowCtx = shadowCaretaker.applyAttributesDefinition(ctx, repoShadow); // TODO: shadowState - repoShadow = shadowManager.updateShadow(shadowCtx, resourceShadow, repoShadow, null, parentResult); + repoShadow = shadowManager.updateShadow(shadowCtx, resourceShadow, repoShadow, null, + null, parentResult); resultShadow = completeShadow(shadowCtx, resourceShadow, repoShadow, isDoDiscovery, objResult); + + // TODO do we want also to futurize the shadow like in getObject? //check and fix kind/intent ShadowType repoShadowType = repoShadow.asObjectable(); diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectAlreadyExistHandler.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectAlreadyExistHandler.java index 0ad45d35cff..24366975a4e 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectAlreadyExistHandler.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectAlreadyExistHandler.java @@ -7,7 +7,6 @@ package com.evolveum.midpoint.provisioning.impl.errorhandling; -import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -29,7 +28,7 @@ import com.evolveum.midpoint.provisioning.impl.ProvisioningContext; import com.evolveum.midpoint.provisioning.impl.ProvisioningOperationState; import com.evolveum.midpoint.provisioning.impl.ShadowCaretaker; -import com.evolveum.midpoint.provisioning.impl.ShadowManager; +import com.evolveum.midpoint.provisioning.impl.shadowmanager.ShadowManager; import com.evolveum.midpoint.provisioning.ucf.api.GenericFrameworkException; import com.evolveum.midpoint.provisioning.util.ProvisioningUtil; import com.evolveum.midpoint.repo.api.RepositoryService; diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectNotFoundHandler.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectNotFoundHandler.java index f2139bcdd75..42ea1ef4bc9 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectNotFoundHandler.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectNotFoundHandler.java @@ -22,7 +22,7 @@ import com.evolveum.midpoint.provisioning.impl.ProvisioningContext; import com.evolveum.midpoint.provisioning.impl.ProvisioningOperationState; import com.evolveum.midpoint.provisioning.impl.ShadowCaretaker; -import com.evolveum.midpoint.provisioning.impl.ShadowManager; +import com.evolveum.midpoint.provisioning.impl.shadowmanager.ShadowManager; import com.evolveum.midpoint.provisioning.impl.ShadowState; import com.evolveum.midpoint.provisioning.ucf.api.GenericFrameworkException; import com.evolveum.midpoint.provisioning.util.ProvisioningUtil; diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadowmanager/ShadowDeltaComputer.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadowmanager/ShadowDeltaComputer.java new file mode 100644 index 00000000000..044e5aeca7e --- /dev/null +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadowmanager/ShadowDeltaComputer.java @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2019 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.provisioning.impl.shadowmanager; + +import com.evolveum.midpoint.common.Clock; +import com.evolveum.midpoint.common.refinery.RefinedAttributeDefinition; +import com.evolveum.midpoint.common.refinery.RefinedObjectClassDefinition; +import com.evolveum.midpoint.prism.*; +import com.evolveum.midpoint.prism.delta.ObjectDelta; +import com.evolveum.midpoint.prism.delta.PropertyDelta; +import com.evolveum.midpoint.prism.match.MatchingRule; +import com.evolveum.midpoint.prism.match.MatchingRuleRegistry; +import com.evolveum.midpoint.prism.path.ItemPath; +import com.evolveum.midpoint.prism.polystring.PolyString; +import com.evolveum.midpoint.provisioning.impl.ProvisioningContext; +import com.evolveum.midpoint.provisioning.impl.ShadowState; +import com.evolveum.midpoint.provisioning.util.ProvisioningUtil; +import com.evolveum.midpoint.schema.constants.SchemaConstants; +import com.evolveum.midpoint.schema.util.ShadowUtil; +import com.evolveum.midpoint.util.exception.*; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.xml.ns._public.common.common_3.CachingMetadataType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.CachingStategyType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType; +import org.jetbrains.annotations.NotNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.xml.namespace.QName; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; + +/** + * Computes deltas to be applied to repository shadows. + * This functionality grew too large to deserve special implementation class. + * + * In the future we might move more functionality here and rename this class. + */ +@Component +public class ShadowDeltaComputer { + + private static final Trace LOGGER = TraceManager.getTrace(ShadowDeltaComputer.class); + + @Autowired private Clock clock; + @Autowired private MatchingRuleRegistry matchingRuleRegistry; + @Autowired private PrismContext prismContext; + + ObjectDelta computeShadowDelta(@NotNull ProvisioningContext ctx, + @NotNull PrismObject repoShadowOld, PrismObject resourceShadowNew, + ObjectDelta explicitShadowDelta, ShadowState shadowState) + throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, + ExpressionEvaluationException { + RefinedObjectClassDefinition ocDef = ctx.computeCompositeObjectClassDefinition(resourceShadowNew); + ObjectDelta computedShadowDelta = repoShadowOld.createModifyDelta(); + PrismContainer currentResourceAttributesContainer = resourceShadowNew.findContainer(ShadowType.F_ATTRIBUTES); + PrismContainer oldRepoAttributesContainer = repoShadowOld.findContainer(ShadowType.F_ATTRIBUTES); + ShadowType oldRepoShadowType = repoShadowOld.asObjectable(); + + CachingStategyType cachingStrategy = ProvisioningUtil.getCachingStrategy(ctx); + + Collection incompleteCacheableItems = new HashSet<>(); + + for (Item currentResourceAttribute: currentResourceAttributesContainer.getValue().getItems()) { + if (currentResourceAttribute instanceof PrismProperty) { + //noinspection unchecked + PrismProperty currentResourceAttrProperty = (PrismProperty) currentResourceAttribute; + RefinedAttributeDefinition attrDef = ocDef.findAttributeDefinition(currentResourceAttrProperty.getElementName()); + if (attrDef == null) { + throw new SchemaException("No definition of " + currentResourceAttrProperty.getElementName() + " in " + ocDef); + } + if (ProvisioningUtil.shouldStoreAttributeInShadow(ocDef, attrDef.getItemName(), cachingStrategy)) { + if (!currentResourceAttribute.isIncomplete()) { + MatchingRule matchingRule = matchingRuleRegistry.getMatchingRule(attrDef.getMatchingRuleQName(), attrDef.getTypeName()); + PrismProperty oldRepoAttributeProperty = oldRepoAttributesContainer.findProperty(currentResourceAttrProperty.getElementName()); + if (oldRepoAttributeProperty == null) { + PropertyDelta attrAddDelta = currentResourceAttrProperty.createDelta(); + List> currentValues = currentResourceAttrProperty.getValues(); + // This is a brutal hack: For extension attributes the ADD operation is slow when using large # of + // values to add. So let's do REPLACE instead (this is OK if there are no existing values). + // TODO Move this logic to repository. Here it is only for PoC purposes. + if (currentValues.size() >= 100) { + Object[] currentValuesNormalized = new Object[currentValues.size()]; + for (int i = 0; i < currentValues.size(); i++) { + currentValuesNormalized[i] = matchingRule.normalize(currentValues.get(i).getValue()); + } + attrAddDelta.setRealValuesToReplace(currentValuesNormalized); + } else { + for (PrismPropertyValue pval : currentValues) { + attrAddDelta.addRealValuesToAdd(matchingRule.normalize(pval.getValue())); + } + } + if (attrAddDelta.getDefinition().getTypeName() == null) { + throw new SchemaException("No definition in " + attrAddDelta); + } + computedShadowDelta.addModification(attrAddDelta); + } else { + if (attrDef.isSingleValue()) { + Object currentResourceRealValue = currentResourceAttrProperty.getRealValue(); + Object currentResourceNormalizedRealValue = matchingRule.normalize(currentResourceRealValue); + if (!Objects.equals(currentResourceNormalizedRealValue, oldRepoAttributeProperty.getRealValue())) { + PropertyDelta delta; + if (currentResourceNormalizedRealValue != null) { + delta = computedShadowDelta.addModificationReplaceProperty(currentResourceAttrProperty.getPath(), + currentResourceNormalizedRealValue); + } else { + delta = computedShadowDelta.addModificationReplaceProperty(currentResourceAttrProperty.getPath()); + } + //noinspection unchecked + delta.setDefinition(currentResourceAttrProperty.getDefinition()); + if (delta.getDefinition().getTypeName() == null) { + throw new SchemaException("No definition in " + delta); + } + } + } else { + PrismProperty normalizedCurrentResourceAttrProperty = currentResourceAttrProperty.clone(); + for (PrismPropertyValue pval : normalizedCurrentResourceAttrProperty.getValues()) { + Object normalizedRealValue = matchingRule.normalize(pval.getValue()); + //noinspection unchecked + pval.setValue(normalizedRealValue); + } + PropertyDelta attrDiff = oldRepoAttributeProperty.diff(normalizedCurrentResourceAttrProperty); + // LOGGER.trace("DIFF:\n{}\n-\n{}\n=:\n{}", + // oldRepoAttributeProperty==null?null:oldRepoAttributeProperty.debugDump(1), + // normalizedCurrentResourceAttrProperty==null?null:normalizedCurrentResourceAttrProperty.debugDump(1), + // attrDiff==null?null:attrDiff.debugDump(1)); + if (attrDiff != null && !attrDiff.isEmpty()) { + attrDiff.setParentPath(ShadowType.F_ATTRIBUTES); + if (attrDiff.getDefinition().getTypeName() == null) { + throw new SchemaException("No definition in " + attrDiff); + } + computedShadowDelta.addModification(attrDiff); + } + } + } + } else { + LOGGER.trace("Resource attribute {} is incomplete, will not update the shadow with its content", + currentResourceAttribute.getElementName()); + incompleteCacheableItems.add(currentResourceAttribute.getElementName()); + } + } else { + LOGGER.trace("Skipping resource attribute because it's not going to be stored in shadow: {}", attrDef.getItemName()); + } + } else { + LOGGER.warn("Skipping resource attribute because it's not a PrismProperty (huh?): {}", currentResourceAttribute); + } + } + + for (Item oldRepoItem: oldRepoAttributesContainer.getValue().getItems()) { + if (oldRepoItem instanceof PrismProperty) { + PrismProperty oldRepoAttrProperty = (PrismProperty)oldRepoItem; + RefinedAttributeDefinition attrDef = ocDef.findAttributeDefinition(oldRepoAttrProperty.getElementName()); + PrismProperty currentAttribute = currentResourceAttributesContainer.findProperty(oldRepoAttrProperty.getElementName()); + // note: incomplete attributes with no values are not here: they are found in currentResourceAttributesContainer + if (attrDef == null || !ProvisioningUtil.shouldStoreAttributeInShadow(ocDef, attrDef.getItemName(), cachingStrategy) || + currentAttribute == null) { + // No definition for this property it should not be there or no current value: remove it from the shadow + PropertyDelta oldRepoAttrPropDelta = oldRepoAttrProperty.createDelta(); + oldRepoAttrPropDelta.addValuesToDelete((Collection) PrismValueCollectionsUtil.cloneCollection(oldRepoAttrProperty.getValues())); + if (oldRepoAttrPropDelta.getDefinition().getTypeName() == null) { + throw new SchemaException("No definition in "+oldRepoAttrPropDelta); + } + computedShadowDelta.addModification(oldRepoAttrPropDelta); + } + } + } + + PolyString currentShadowName = ShadowUtil.determineShadowName(resourceShadowNew); + PolyString oldRepoShadowName = repoShadowOld.getName(); + if (!currentShadowName.equalsOriginalValue(oldRepoShadowName)) { + PropertyDelta shadowNameDelta = prismContext.deltaFactory().property().createModificationReplaceProperty(ShadowType.F_NAME, + repoShadowOld.getDefinition(),currentShadowName); + computedShadowDelta.addModification(shadowNameDelta); + } + + PropertyDelta auxOcDelta = ItemUtil.diff( + repoShadowOld.findProperty(ShadowType.F_AUXILIARY_OBJECT_CLASS), + resourceShadowNew.findProperty(ShadowType.F_AUXILIARY_OBJECT_CLASS)); + if (auxOcDelta != null) { + computedShadowDelta.addModification(auxOcDelta); + } + + // Resource object obviously exists in this case. However, we do not want to mess with isExists flag in some + // situations (e.g. in CORPSE state) as this existence may be just a quantum illusion. + if (shadowState == ShadowState.CONCEPTION || shadowState == ShadowState.GESTATION) { + PropertyDelta existsDelta = computedShadowDelta.createPropertyModification(ShadowType.F_EXISTS); + existsDelta.setRealValuesToReplace(true); + computedShadowDelta.addModification(existsDelta); + } + + if (cachingStrategy == CachingStategyType.NONE) { + if (oldRepoShadowType.getCachingMetadata() != null) { + computedShadowDelta.addModificationReplaceProperty(ShadowType.F_CACHING_METADATA); + } + + } else if (cachingStrategy == CachingStategyType.PASSIVE) { + + compareUpdateProperty(computedShadowDelta, SchemaConstants.PATH_ACTIVATION_ADMINISTRATIVE_STATUS, resourceShadowNew, repoShadowOld); + compareUpdateProperty(computedShadowDelta, SchemaConstants.PATH_ACTIVATION_VALID_FROM, resourceShadowNew, repoShadowOld); + compareUpdateProperty(computedShadowDelta, SchemaConstants.PATH_ACTIVATION_VALID_TO, resourceShadowNew, repoShadowOld); + compareUpdateProperty(computedShadowDelta, SchemaConstants.PATH_ACTIVATION_LOCKOUT_STATUS, resourceShadowNew, repoShadowOld); + + if (incompleteCacheableItems.isEmpty()) { + CachingMetadataType cachingMetadata = new CachingMetadataType(); + cachingMetadata.setRetrievalTimestamp(clock.currentTimeXMLGregorianCalendar()); + computedShadowDelta.addModificationReplaceProperty(ShadowType.F_CACHING_METADATA, cachingMetadata); + } else { + LOGGER.trace("Shadow has incomplete cacheable items; will not update caching timestamp: {}", incompleteCacheableItems); + } + } else { + throw new ConfigurationException("Unknown caching strategy "+cachingStrategy); + } + return computedShadowDelta; + } + + private void compareUpdateProperty(ObjectDelta shadowDelta, + ItemPath itemPath, PrismObject currentResourceShadow, PrismObject oldRepoShadow) { + PrismProperty currentProperty = currentResourceShadow.findProperty(itemPath); + PrismProperty oldProperty = oldRepoShadow.findProperty(itemPath); + PropertyDelta itemDelta = ItemUtil.diff(oldProperty, currentProperty); + if (itemDelta != null && !itemDelta.isEmpty()) { + shadowDelta.addModification(itemDelta); + } + } +} diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadowmanager/ShadowManager.java similarity index 90% rename from provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java rename to provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadowmanager/ShadowManager.java index a5438c9f778..6f08ba94fc5 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadowmanager/ShadowManager.java @@ -5,7 +5,7 @@ * and European Union Public License. See LICENSE file for details. */ -package com.evolveum.midpoint.provisioning.impl; +package com.evolveum.midpoint.provisioning.impl.shadowmanager; import java.util.*; import java.util.Objects; @@ -18,6 +18,10 @@ import com.evolveum.midpoint.prism.path.ItemPath; import com.evolveum.midpoint.prism.path.ItemName; import com.evolveum.midpoint.provisioning.api.ProvisioningService; +import com.evolveum.midpoint.provisioning.impl.ConstraintsChecker; +import com.evolveum.midpoint.provisioning.impl.ProvisioningContext; +import com.evolveum.midpoint.provisioning.impl.ProvisioningOperationState; +import com.evolveum.midpoint.provisioning.impl.ShadowState; import com.evolveum.midpoint.schema.*; import com.evolveum.midpoint.xml.ns._public.common.common_3.*; import org.apache.commons.lang.BooleanUtils; @@ -103,6 +107,7 @@ public class ShadowManager { @Autowired private MatchingRuleRegistry matchingRuleRegistry; @Autowired private Protector protector; @Autowired private ProvisioningService provisioningService; + @Autowired private ShadowDeltaComputer shadowDeltaComputer; private static final Trace LOGGER = TraceManager.getTrace(ShadowManager.class); @@ -198,7 +203,7 @@ public PrismObject lookupShadowByPrimaryIdentifierValue(Provisioning } if (foundShadows.size() > 1) { // This cannot happen, there is an unique constraint on primaryIdentifierValue - LOGGER.error("Impossible just happened, found {} shadows for primaryIdentifierValue {}: {}", foundShadows.size(), primaryIdentifierValue); + LOGGER.error("Impossible just happened, found {} shadows for primaryIdentifierValue {}: {}", foundShadows.size(), primaryIdentifierValue, foundShadows); throw new SystemException("Impossible just happened, found "+foundShadows.size()+" shadows for primaryIdentifierValue "+primaryIdentifierValue); } @@ -701,7 +706,7 @@ public PrismObject addDiscoveredRepositoryShadow(ProvisioningContext } public void addNewProposedShadow(ProvisioningContext ctx, PrismObject shadowToAdd, - ProvisioningOperationState>> opState, + ProvisioningOperationState>> opState, Task task, OperationResult result) throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, ExpressionEvaluationException, ObjectAlreadyExistsException, SecurityViolationException, EncryptionException { if (!isUseProposedShadows(ctx)) { return; @@ -814,9 +819,6 @@ private void recordAddResultNewShadow( parentResult.recordSuccess(); } - - - @SuppressWarnings("unchecked") private void recordAddResultExistingShadow( ProvisioningContext ctx, PrismObject shadowToAdd, @@ -1769,7 +1771,7 @@ private void hashValues(Collection> pval } } - @SuppressWarnings("unchecked") + // TODO remove this method if really not needed public Collection updateShadow(ProvisioningContext ctx, PrismObject resourceShadow, Collection aprioriDeltas, OperationResult result) throws SchemaException, ConfigurationException, ObjectNotFoundException, CommunicationException, ExpressionEvaluationException { PrismObject repoShadow = repositoryService.getObject(ShadowType.class, resourceShadow.getOid(), null, result); @@ -1778,9 +1780,7 @@ public Collection updateShadow(ProvisioningContext ctx, PrismObject updateShadow(@NotNull ProvisioningContext ctx, @NotNull PrismObject resourceShadowNew, - @NotNull PrismObject repoShadowOld, ShadowState shadowState, OperationResult parentResult) throws SchemaException, + @NotNull PrismObject repoShadowOld, + ObjectDelta explicitShadowDelta, + ShadowState shadowState, OperationResult parentResult) throws SchemaException, ObjectNotFoundException, ConfigurationException, CommunicationException, ExpressionEvaluationException { - ObjectDelta shadowDelta = computeShadowDelta(ctx, repoShadowOld, resourceShadowNew, shadowState); + ObjectDelta computedShadowDelta = shadowDeltaComputer.computeShadowDelta(ctx, repoShadowOld, resourceShadowNew, + explicitShadowDelta, shadowState); - if (!shadowDelta.isEmpty()) { - LOGGER.trace("Updating repo shadow {} with delta:\n{}", repoShadowOld, shadowDelta.debugDumpLazily(1)); - ConstraintsChecker.onShadowModifyOperation(shadowDelta.getModifications()); + if (!computedShadowDelta.isEmpty()) { + LOGGER.trace("Updating repo shadow {} with delta:\n{}", repoShadowOld, computedShadowDelta.debugDumpLazily(1)); + ConstraintsChecker.onShadowModifyOperation(computedShadowDelta.getModifications()); try { - repositoryService.modifyObject(ShadowType.class, repoShadowOld.getOid(), shadowDelta.getModifications(), parentResult); + repositoryService.modifyObject(ShadowType.class, repoShadowOld.getOid(), computedShadowDelta.getModifications(), parentResult); } catch (ObjectAlreadyExistsException e) { throw new SystemException(e.getMessage(), e); // This should not happen for shadows } PrismObject repoShadowNew = repoShadowOld.clone(); - shadowDelta.applyTo(repoShadowNew); + computedShadowDelta.applyTo(repoShadowNew); return repoShadowNew; } else { LOGGER.trace("No need to update repo shadow {} (empty delta)", repoShadowOld); @@ -1861,179 +1866,6 @@ public PrismObject updateShadow(@NotNull ProvisioningContext ctx, } } - private ObjectDelta computeShadowDelta(@NotNull ProvisioningContext ctx, - @NotNull PrismObject repoShadowOld, - PrismObject resourceShadowNew, ShadowState shadowState) - throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, - ExpressionEvaluationException { - RefinedObjectClassDefinition ocDef = ctx.computeCompositeObjectClassDefinition(resourceShadowNew); - ObjectDelta shadowDelta = repoShadowOld.createModifyDelta(); - PrismContainer currentResourceAttributesContainer = resourceShadowNew.findContainer(ShadowType.F_ATTRIBUTES); - PrismContainer oldRepoAttributesContainer = repoShadowOld.findContainer(ShadowType.F_ATTRIBUTES); - ShadowType oldRepoShadowType = repoShadowOld.asObjectable(); - - CachingStategyType cachingStrategy = ProvisioningUtil.getCachingStrategy(ctx); - - Collection incompleteCacheableItems = new HashSet<>(); - - for (Item currentResourceItem: currentResourceAttributesContainer.getValue().getItems()) { - if (currentResourceItem instanceof PrismProperty) { - //noinspection unchecked - PrismProperty currentResourceAttrProperty = (PrismProperty) currentResourceItem; - RefinedAttributeDefinition attrDef = ocDef.findAttributeDefinition(currentResourceAttrProperty.getElementName()); - if (attrDef == null) { - throw new SchemaException("No definition of " + currentResourceAttrProperty.getElementName() + " in " + ocDef); - } - if (ProvisioningUtil.shouldStoreAttributeInShadow(ocDef, attrDef.getItemName(), cachingStrategy)) { - if (!currentResourceItem.isIncomplete()) { - MatchingRule matchingRule = matchingRuleRegistry.getMatchingRule(attrDef.getMatchingRuleQName(), attrDef.getTypeName()); - PrismProperty oldRepoAttributeProperty = oldRepoAttributesContainer.findProperty(currentResourceAttrProperty.getElementName()); - if (oldRepoAttributeProperty == null) { - PropertyDelta attrAddDelta = currentResourceAttrProperty.createDelta(); - List> currentValues = currentResourceAttrProperty.getValues(); - // This is a brutal hack: For extension attributes the ADD operation is slow when using large # of - // values to add. So let's do REPLACE instead (this is OK if there are no existing values). - // TODO Move this logic to repository. Here it is only for PoC purposes. - if (currentValues.size() >= 100) { - Object[] currentValuesNormalized = new Object[currentValues.size()]; - for (int i = 0; i < currentValues.size(); i++) { - currentValuesNormalized[i] = matchingRule.normalize(currentValues.get(i).getValue()); - } - attrAddDelta.setRealValuesToReplace(currentValuesNormalized); - } else { - for (PrismPropertyValue pval : currentValues) { - attrAddDelta.addRealValuesToAdd(matchingRule.normalize(pval.getValue())); - } - } - if (attrAddDelta.getDefinition().getTypeName() == null) { - throw new SchemaException("No definition in " + attrAddDelta); - } - shadowDelta.addModification(attrAddDelta); - } else { - if (attrDef.isSingleValue()) { - Object currentResourceRealValue = currentResourceAttrProperty.getRealValue(); - Object currentResourceNormalizedRealValue = matchingRule.normalize(currentResourceRealValue); - if (!Objects.equals(currentResourceNormalizedRealValue, oldRepoAttributeProperty.getRealValue())) { - PropertyDelta delta; - if (currentResourceNormalizedRealValue != null) { - delta = shadowDelta.addModificationReplaceProperty(currentResourceAttrProperty.getPath(), - currentResourceNormalizedRealValue); - } else { - delta = shadowDelta.addModificationReplaceProperty(currentResourceAttrProperty.getPath()); - } - //noinspection unchecked - delta.setDefinition(currentResourceAttrProperty.getDefinition()); - if (delta.getDefinition().getTypeName() == null) { - throw new SchemaException("No definition in " + delta); - } - } - } else { - PrismProperty normalizedCurrentResourceAttrProperty = currentResourceAttrProperty.clone(); - for (PrismPropertyValue pval : normalizedCurrentResourceAttrProperty.getValues()) { - Object normalizedRealValue = matchingRule.normalize(pval.getValue()); - //noinspection unchecked - pval.setValue(normalizedRealValue); - } - PropertyDelta attrDiff = oldRepoAttributeProperty.diff(normalizedCurrentResourceAttrProperty); -// LOGGER.trace("DIFF:\n{}\n-\n{}\n=:\n{}", -// oldRepoAttributeProperty==null?null:oldRepoAttributeProperty.debugDump(1), -// normalizedCurrentResourceAttrProperty==null?null:normalizedCurrentResourceAttrProperty.debugDump(1), -// attrDiff==null?null:attrDiff.debugDump(1)); - if (attrDiff != null && !attrDiff.isEmpty()) { - attrDiff.setParentPath(ShadowType.F_ATTRIBUTES); - if (attrDiff.getDefinition().getTypeName() == null) { - throw new SchemaException("No definition in " + attrDiff); - } - shadowDelta.addModification(attrDiff); - } - } - } - } else { - LOGGER.trace("Resource item {} is incomplete, will not update the shadow with its content", - currentResourceItem.getElementName()); - incompleteCacheableItems.add(currentResourceItem.getElementName()); - } - } - } - } - - for (Item oldRepoItem: oldRepoAttributesContainer.getValue().getItems()) { - if (oldRepoItem instanceof PrismProperty) { - PrismProperty oldRepoAttrProperty = (PrismProperty)oldRepoItem; - RefinedAttributeDefinition attrDef = ocDef.findAttributeDefinition(oldRepoAttrProperty.getElementName()); - PrismProperty currentAttribute = currentResourceAttributesContainer.findProperty(oldRepoAttrProperty.getElementName()); - // note: incomplete attributes with no values are not here: they are found in currentResourceAttributesContainer - if (attrDef == null || !ProvisioningUtil.shouldStoreAttributeInShadow(ocDef, attrDef.getItemName(), cachingStrategy) || - currentAttribute == null) { - // No definition for this property it should not be there or no current value: remove it from the shadow - PropertyDelta oldRepoAttrPropDelta = oldRepoAttrProperty.createDelta(); - oldRepoAttrPropDelta.addValuesToDelete((Collection) PrismValueCollectionsUtil.cloneCollection(oldRepoAttrProperty.getValues())); - if (oldRepoAttrPropDelta.getDefinition().getTypeName() == null) { - throw new SchemaException("No definition in "+oldRepoAttrPropDelta); - } - shadowDelta.addModification(oldRepoAttrPropDelta); - } - } - } - - PolyString currentShadowName = ShadowUtil.determineShadowName(resourceShadowNew); - PolyString oldRepoShadowName = repoShadowOld.getName(); - if (!currentShadowName.equalsOriginalValue(oldRepoShadowName)) { - PropertyDelta shadowNameDelta = prismContext.deltaFactory().property().createModificationReplaceProperty(ShadowType.F_NAME, - repoShadowOld.getDefinition(),currentShadowName); - shadowDelta.addModification(shadowNameDelta); - } - - PropertyDelta auxOcDelta = ItemUtil.diff( - repoShadowOld.findProperty(ShadowType.F_AUXILIARY_OBJECT_CLASS), - resourceShadowNew.findProperty(ShadowType.F_AUXILIARY_OBJECT_CLASS)); - if (auxOcDelta != null) { - shadowDelta.addModification(auxOcDelta); - } - - // Resource object obviously exists in this case. However, we do not want to mess with isExists flag in some - // situations (e.g. in CORPSE state) as this existence may be just a quantum illusion. - if (shadowState == ShadowState.CONCEPTION || shadowState == ShadowState.GESTATION) { - PropertyDelta existsDelta = shadowDelta.createPropertyModification(ShadowType.F_EXISTS); - existsDelta.setRealValuesToReplace(true); - shadowDelta.addModification(existsDelta); - } - - if (cachingStrategy == CachingStategyType.NONE) { - if (oldRepoShadowType.getCachingMetadata() != null) { - shadowDelta.addModificationReplaceProperty(ShadowType.F_CACHING_METADATA); - } - - } else if (cachingStrategy == CachingStategyType.PASSIVE) { - - compareUpdateProperty(shadowDelta, SchemaConstants.PATH_ACTIVATION_ADMINISTRATIVE_STATUS, resourceShadowNew, repoShadowOld); - compareUpdateProperty(shadowDelta, SchemaConstants.PATH_ACTIVATION_VALID_FROM, resourceShadowNew, repoShadowOld); - compareUpdateProperty(shadowDelta, SchemaConstants.PATH_ACTIVATION_VALID_TO, resourceShadowNew, repoShadowOld); - compareUpdateProperty(shadowDelta, SchemaConstants.PATH_ACTIVATION_LOCKOUT_STATUS, resourceShadowNew, repoShadowOld); - - if (incompleteCacheableItems.isEmpty()) { - CachingMetadataType cachingMetadata = new CachingMetadataType(); - cachingMetadata.setRetrievalTimestamp(clock.currentTimeXMLGregorianCalendar()); - shadowDelta.addModificationReplaceProperty(ShadowType.F_CACHING_METADATA, cachingMetadata); - } else { - LOGGER.trace("Shadow has incomplete cacheable items; will not update caching timestamp: {}", incompleteCacheableItems); - } - } else { - throw new ConfigurationException("Unknown caching strategy "+cachingStrategy); - } - return shadowDelta; - } - - private void compareUpdateProperty(ObjectDelta shadowDelta, - ItemPath itemPath, PrismObject currentResourceShadow, PrismObject oldRepoShadow) { - PrismProperty currentProperty = currentResourceShadow.findProperty(itemPath); - PrismProperty oldProperty = oldRepoShadow.findProperty(itemPath); - PropertyDelta itemDelta = ItemUtil.diff(oldProperty, currentProperty); - if (itemDelta != null && !itemDelta.isEmpty()) { - shadowDelta.addModification(itemDelta); - } - } - public PrismObject recordDeleteResult( ProvisioningContext ctx, PrismObject oldRepoShadow, diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/sync/ChangeProcessor.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/sync/ChangeProcessor.java index 1160f0444cf..f8d9fcd274a 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/sync/ChangeProcessor.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/sync/ChangeProcessor.java @@ -15,7 +15,7 @@ import com.evolveum.midpoint.provisioning.impl.ProvisioningContext; import com.evolveum.midpoint.provisioning.impl.ShadowCache; import com.evolveum.midpoint.provisioning.impl.ShadowCaretaker; -import com.evolveum.midpoint.provisioning.impl.ShadowManager; +import com.evolveum.midpoint.provisioning.impl.shadowmanager.ShadowManager; import com.evolveum.midpoint.provisioning.ucf.api.Change; import com.evolveum.midpoint.provisioning.util.ProvisioningUtil; import com.evolveum.midpoint.repo.api.PreconditionViolationException; @@ -260,7 +260,7 @@ private void preProcessChange(ProvisioningContext ctx, Change change, OperationR PrismObject currentShadow = shadowCache.completeShadow(ctx, change.getCurrentShadow(), oldShadow, false, parentResult); change.setCurrentShadow(currentShadow); // TODO: shadowState - shadowManager.updateShadow(ctx, currentShadow, oldShadow, null, parentResult); + shadowManager.updateShadow(ctx, currentShadow, oldShadow, change.getObjectDelta(), null, parentResult); } if (change.getObjectDelta() != null && change.getObjectDelta().getOid() == null) { diff --git a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdate.java b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdate.java index db6e0554e57..69042c0c718 100644 --- a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdate.java +++ b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdate.java @@ -15,7 +15,9 @@ import com.evolveum.midpoint.prism.delta.ItemDelta; import com.evolveum.midpoint.provisioning.api.ResourceObjectShadowChangeDescription; import com.evolveum.midpoint.provisioning.impl.AbstractProvisioningIntegrationTest; +import com.evolveum.midpoint.schema.GetOperationOptions; import com.evolveum.midpoint.schema.ResourceShadowDiscriminator; +import com.evolveum.midpoint.schema.SelectorOptions; import com.evolveum.midpoint.schema.constants.MidPointConstants; import com.evolveum.midpoint.schema.internals.InternalsConfig; import com.evolveum.midpoint.schema.processor.ObjectClassComplexTypeDefinition; @@ -26,11 +28,13 @@ import com.evolveum.midpoint.schema.util.ResourceTypeUtil; import com.evolveum.midpoint.task.api.Task; import com.evolveum.midpoint.test.IntegrationTestTools; +import com.evolveum.midpoint.test.asserter.ShadowAsserter; import com.evolveum.midpoint.test.util.TestUtil; -import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.util.exception.*; import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; import com.evolveum.midpoint.xml.ns._public.common.common_3.*; +import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.NotNull; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; @@ -47,7 +51,6 @@ import java.util.concurrent.TimeoutException; import static com.evolveum.midpoint.provisioning.impl.ProvisioningTestUtil.checkRepoAccountShadow; -import static com.evolveum.midpoint.provisioning.impl.ProvisioningTestUtil.checkRepoShadow; import static org.testng.AssertJUnit.*; /** @@ -67,7 +70,9 @@ public abstract class TestAsyncUpdate extends AbstractProvisioningIntegrationTes private static final String RESOURCE_ASYNC_OID = "fb04d113-ebf8-41b4-b13b-990a597d110b"; private static final File CHANGE_100 = new File(TEST_DIR, "change-100-banderson-first-occurrence.xml"); - private static final File CHANGE_110 = new File(TEST_DIR, "change-110-banderson-delta.xml"); + private static final File CHANGE_110 = new File(TEST_DIR, "change-110-banderson-delta-add-values.xml"); + private static final File CHANGE_115 = new File(TEST_DIR, "change-115-banderson-delta-delete-values.xml"); + private static final File CHANGE_117 = new File(TEST_DIR, "change-117-banderson-delta-replace-values.xml"); private static final File CHANGE_120 = new File(TEST_DIR, "change-120-banderson-new-state.xml"); private static final File CHANGE_125 = new File(TEST_DIR, "change-125-banderson-notification-only.xml"); private static final File CHANGE_130 = new File(TEST_DIR, "change-130-banderson-delete.xml"); @@ -80,6 +85,8 @@ public abstract class TestAsyncUpdate extends AbstractProvisioningIntegrationTes private static final Trace LOGGER = TraceManager.getTrace(TestAsyncUpdate.class); private static final long TIMEOUT = 5000L; + public static final String ATTR_TEST = "test"; + public static final String ATTR_MEMBER_OF = "memberOf"; protected PrismObject resource; @@ -297,10 +304,94 @@ public void test110ListeningForValueAdd(ITestContext ctx) throws Exception { assertEquals("Wrong # of values added (second mod)", 6, iterator.next().getValuesToAdd().size()); assertNotNull("Current shadow is not present", lastChange.getCurrentShadow()); - PrismObject accountRepo = findAccountShadowByUsername("banderson", resource, result); - assertNotNull("Shadow is not present in the repository", accountRepo); - display("Repository shadow", accountRepo); - checkRepoShadow(accountRepo, ShadowKindType.ACCOUNT, getNumberOfAccountAttributes()); + ShadowAsserter asserter = getAndersonFull(false, task, result); + if (isCached()) { + asserter.attributes() + .attribute(ATTR_TEST).assertRealValues("value1", "value2", "value3").end() + .attribute(ATTR_MEMBER_OF).assertRealValues("group1", "group2", "group3", "group4", "group5", "group6").end(); + } + } + + @Test // MID-5832 + public void test115ListeningForValueDelete(ITestContext ctx) throws Exception { + Task task = getTask(ctx); + OperationResult result = getResult(ctx); + + prepareMessage(CHANGE_115); + + syncServiceMock.reset(); + + setDummyAccountTestAttribute("banderson", "value1", "value3"); + + ResourceShadowDiscriminator coords = new ResourceShadowDiscriminator(RESOURCE_ASYNC_OID); + String handle = provisioningService.startListeningForAsyncUpdates(coords, task, result); + dumpAsyncUpdateListeningActivity(handle, task, result); + syncServiceMock.waitForNotifyChange(TIMEOUT); + provisioningService.stopListeningForAsyncUpdates(handle, task, result); + + ResourceObjectShadowChangeDescription lastChange = syncServiceMock.getLastChange(); + display("The change", lastChange); + + PrismObject oldShadow = lastChange.getOldShadow(); + assertNotNull("Old shadow missing", oldShadow); + assertNotNull("Old shadow does not have an OID", oldShadow.getOid()); + + assertNotNull("Delta is missing", lastChange.getObjectDelta()); + assertTrue("Delta is not a MODIFY one", lastChange.getObjectDelta().isModify()); + Collection> modifications = lastChange.getObjectDelta().getModifications(); + assertEquals("Wrong # of modifications", 2, modifications.size()); + Iterator> iterator = modifications.iterator(); + assertEquals("Wrong # of values deleted (first mod)", 1, iterator.next().getValuesToDelete().size()); + assertEquals("Wrong # of values deleted (second mod)", 2, iterator.next().getValuesToDelete().size()); + assertNotNull("Current shadow is not present", lastChange.getCurrentShadow()); + + ShadowAsserter asserter = getAndersonFull(false, task, result); + if (isCached()) { + asserter.attributes() + .attribute(ATTR_TEST).assertRealValues("value1", "value3").end() + .attribute(ATTR_MEMBER_OF).assertRealValues("group1", "group4", "group5", "group6").end(); + } + } + + @Test // MID-5832 + public void test117ListeningForValueReplace(ITestContext ctx) throws Exception { + Task task = getTask(ctx); + OperationResult result = getResult(ctx); + + prepareMessage(CHANGE_117); + + syncServiceMock.reset(); + + setDummyAccountTestAttribute("banderson", "value100"); + + ResourceShadowDiscriminator coords = new ResourceShadowDiscriminator(RESOURCE_ASYNC_OID); + String handle = provisioningService.startListeningForAsyncUpdates(coords, task, result); + dumpAsyncUpdateListeningActivity(handle, task, result); + syncServiceMock.waitForNotifyChange(TIMEOUT); + provisioningService.stopListeningForAsyncUpdates(handle, task, result); + + ResourceObjectShadowChangeDescription lastChange = syncServiceMock.getLastChange(); + display("The change", lastChange); + + PrismObject oldShadow = lastChange.getOldShadow(); + assertNotNull("Old shadow missing", oldShadow); + assertNotNull("Old shadow does not have an OID", oldShadow.getOid()); + + assertNotNull("Delta is missing", lastChange.getObjectDelta()); + assertTrue("Delta is not a MODIFY one", lastChange.getObjectDelta().isModify()); + Collection> modifications = lastChange.getObjectDelta().getModifications(); + assertEquals("Wrong # of modifications", 2, modifications.size()); + Iterator> iterator = modifications.iterator(); + assertEquals("Wrong # of values replaced (first mod)", 1, iterator.next().getValuesToReplace().size()); + assertEquals("Wrong # of values replaced (second mod)", 2, iterator.next().getValuesToReplace().size()); + assertNotNull("Current shadow is not present", lastChange.getCurrentShadow()); + + ShadowAsserter asserter = getAndersonFull(false, task, result); + if (isCached()) { + asserter.attributes() + .attribute(ATTR_TEST).assertRealValues("value100").end() + .attribute(ATTR_MEMBER_OF).assertRealValues("group100", "group101").end(); + } } @Test @@ -330,10 +421,7 @@ public void test120ListeningForShadowReplace(ITestContext ctx) throws Exception assertNull("Delta is present although it should not be", lastChange.getObjectDelta()); assertNotNull("Current shadow is missing", lastChange.getCurrentShadow()); - PrismObject accountRepo = findAccountShadowByUsername("banderson", resource, result); - assertNotNull("Shadow is not present in the repository", accountRepo); - display("Repository shadow", accountRepo); - checkRepoShadow(accountRepo, ShadowKindType.ACCOUNT, getNumberOfAccountAttributes()); + ShadowAsserter asserter = getAndersonFull(false, task, result); } @Test @@ -370,10 +458,7 @@ public void test125ListeningForNotificationOnly(ITestContext ctx) throws Excepti display("change current shadow", lastChange.getCurrentShadow()); - PrismObject accountRepo = findAccountShadowByUsername("banderson", resource, result); - assertNotNull("Shadow is not present in the repository", accountRepo); - display("Repository shadow", accountRepo); - checkRepoShadow(accountRepo, ShadowKindType.ACCOUNT, getNumberOfAccountAttributes()); + ShadowAsserter asserter = getAndersonFull(false, task, result); } @Test @@ -403,10 +488,7 @@ public void test130ListeningForShadowDelete(ITestContext ctx) throws Exception { //assertNull("Current shadow is present while not expecting it", lastChange.getCurrentShadow()); //current shadow was added during the processing - PrismObject accountRepo = findAccountShadowByUsername("banderson", resource, result); - assertNotNull("Shadow is not present in the repository", accountRepo); - display("Repository shadow", accountRepo); - checkRepoShadow(accountRepo, ShadowKindType.ACCOUNT, getNumberOfAccountAttributes()); + getAndersonFull(true, task, result); } @SuppressWarnings("SameParameterValue") @@ -417,7 +499,11 @@ void addDummyAccount(String name) { void setDummyAccountTestAttribute(String name, String... values) { } - abstract int getNumberOfAccountAttributes(); + private int getNumberOfAccountAttributes() { + return isCached() ? 4 : 2; + } + + abstract boolean isCached(); boolean hasReadCapability() { return false; @@ -428,4 +514,40 @@ void prepareMessage(File messageFile) MockAsyncUpdateSource.INSTANCE.reset(); MockAsyncUpdateSource.INSTANCE.prepareMessage(prismContext.parserFor(messageFile).parseRealValue()); } + + @Contract("false,_,_ -> !null") + private ShadowAsserter getAndersonFull(boolean dead, Task task, OperationResult result) + throws SchemaException, ObjectNotFoundException, SecurityViolationException, CommunicationException, + ConfigurationException, ExpressionEvaluationException { + PrismObject shadowRepo = findAccountShadowByUsername("banderson", resource, result); + assertNotNull("No Anderson shadow in repo", shadowRepo); + Collection> options = schemaHelper.getOperationOptionsBuilder() + .noFetch() + .retrieve() + .build(); + try { + PrismObject shadow = provisioningService + .getObject(ShadowType.class, shadowRepo.getOid(), options, task, result); + if (dead) { + fail("Shadow should be gone now but it is not: " + shadow.debugDump()); + } + return assertShadow(shadow, "after") + .assertKind(ShadowKindType.ACCOUNT) + .attributes() + .assertSize(getNumberOfAccountAttributes()) + .primaryIdentifier() + .assertRealValues("banderson") + .end() + .secondaryIdentifier() + .assertRealValues("banderson") + .end() + .end(); + } catch (ObjectNotFoundException e) { + if (!dead) { + e.printStackTrace(); + fail("Shadow is gone but it should not be"); + } + return null; + } + } } diff --git a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateCaching.java b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateCaching.java index 9f677ab88ed..3ffcf680c6b 100644 --- a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateCaching.java +++ b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateCaching.java @@ -31,7 +31,7 @@ public List getConnectorTypes() { } @Override - protected int getNumberOfAccountAttributes() { - return 4; + boolean isCached() { + return true; } } diff --git a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateCachingIndexOnly.java b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateCachingIndexOnly.java index 5aac38ad198..ba4f2ac3711 100644 --- a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateCachingIndexOnly.java +++ b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateCachingIndexOnly.java @@ -29,11 +29,6 @@ protected File getResourceFile() { return RESOURCE_ASYNC_CACHING_INDEX_ONLY_FILE; } - @Override - protected int getNumberOfAccountAttributes() { - return 3; - } - @Override public void initSystem(Task initTask, OperationResult initResult) throws Exception { // These are experimental features, so they need to be explicitly enabled. This will be eliminated later, diff --git a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateNoCaching.java b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateNoCaching.java index b07c47f6cc1..3f8c69bc6bc 100644 --- a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateNoCaching.java +++ b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/async/TestAsyncUpdateNoCaching.java @@ -49,8 +49,8 @@ public List getConnectorTypes() { } @Override - protected int getNumberOfAccountAttributes() { - return 2; + boolean isCached() { + return false; } @Override diff --git a/provisioning/provisioning-impl/src/test/resources/async/change-110-banderson-delta.xml b/provisioning/provisioning-impl/src/test/resources/async/change-110-banderson-delta-add-values.xml similarity index 100% rename from provisioning/provisioning-impl/src/test/resources/async/change-110-banderson-delta.xml rename to provisioning/provisioning-impl/src/test/resources/async/change-110-banderson-delta-add-values.xml diff --git a/provisioning/provisioning-impl/src/test/resources/async/change-115-banderson-delta-delete-values.xml b/provisioning/provisioning-impl/src/test/resources/async/change-115-banderson-delta-delete-values.xml new file mode 100644 index 00000000000..78bf974178e --- /dev/null +++ b/provisioning/provisioning-impl/src/test/resources/async/change-115-banderson-delta-delete-values.xml @@ -0,0 +1,33 @@ + + + + ri:AccountObjectClass + + banderson + banderson + + + modify + ShadowType + + delete + attributes/ri:test + value2 + + + delete + attributes/ri:memberOf + group2 + group3 + + + diff --git a/provisioning/provisioning-impl/src/test/resources/async/change-117-banderson-delta-replace-values.xml b/provisioning/provisioning-impl/src/test/resources/async/change-117-banderson-delta-replace-values.xml new file mode 100644 index 00000000000..dba7dc494d2 --- /dev/null +++ b/provisioning/provisioning-impl/src/test/resources/async/change-117-banderson-delta-replace-values.xml @@ -0,0 +1,33 @@ + + + + ri:AccountObjectClass + + banderson + banderson + + + modify + ShadowType + + replace + attributes/ri:test + value100 + + + replace + attributes/ri:memberOf + group100 + group101 + + + diff --git a/provisioning/provisioning-impl/testng-integration.xml b/provisioning/provisioning-impl/testng-integration.xml index 97b34a0a114..a9a031cd81d 100644 --- a/provisioning/provisioning-impl/testng-integration.xml +++ b/provisioning/provisioning-impl/testng-integration.xml @@ -67,6 +67,7 @@ + diff --git a/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java b/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java index 3fe9b2c35d8..db8a0fc6cdb 100644 --- a/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java +++ b/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java @@ -242,8 +242,8 @@ public void setTaskAndResult(ITestContext ctx, Method testMethod) throws SchemaE return; } - String testShortName = testMethod.getName(); - String testFullName = testMethod.getDeclaringClass().getName() + "." + testShortName; + String testShortName = testMethod.getDeclaringClass().getSimpleName() + "." + testMethod.getName(); + String testFullName = testMethod.getDeclaringClass().getName() + "." + testMethod.getName(); TestUtil.displayTestTitle(testShortName); Task task = createTask(testFullName); OperationResult rootResult = task.getResult(); diff --git a/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/asserter/ShadowAttributesAsserter.java b/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/asserter/ShadowAttributesAsserter.java index 3eb22b3ff42..08b1ad18eeb 100644 --- a/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/asserter/ShadowAttributesAsserter.java +++ b/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/asserter/ShadowAttributesAsserter.java @@ -29,6 +29,7 @@ import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowAttributesType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType; import com.evolveum.prism.xml.ns._public.types_3.RawType; +import org.apache.commons.collections4.CollectionUtils; /** * @author semancik @@ -90,13 +91,21 @@ public ShadowAttributesAsserter assertAttributes(QName... expectedAttributes) } // TODO: change to ShadowAttributeAsserter later - public PrismPropertyAsserter> attribute(String attrName) { - PrismProperty attribute = findAttribute(attrName); - PrismPropertyAsserter> asserter = new PrismPropertyAsserter<>(attribute, this, "attribute "+attrName+" in "+desc()); - copySetupTo(asserter); - return asserter; - } + public PrismPropertyAsserter> attribute(String attrName) { + PrismProperty attribute = findAttribute(attrName); + PrismPropertyAsserter> asserter = new PrismPropertyAsserter<>(attribute, this, "attribute "+attrName+" in "+desc()); + copySetupTo(asserter); + return asserter; + } + // TODO: change to ShadowAttributeAsserter later + public PrismPropertyAsserter> attribute(QName attrName) { + PrismProperty attribute = findAttribute(attrName); + PrismPropertyAsserter> asserter = new PrismPropertyAsserter<>(attribute, this, "attribute "+attrName+" in "+desc()); + copySetupTo(asserter); + return asserter; + } + public ShadowAttributesAsserter assertAny() { assertNotNull("No attributes container in "+desc(), getAttributesContainer()); PrismContainerValue containerValue = getAttributesContainer().getValue(); @@ -117,27 +126,41 @@ private String presentAttributeNames() { return sb.toString(); } + public PrismPropertyAsserter> primaryIdentifier() { + Collection> primaryIdentifiers = ShadowUtil.getPrimaryIdentifiers(getShadow()); + assertFalse("No primary identifier in "+desc(), CollectionUtils.isEmpty(primaryIdentifiers)); + assertEquals("Wrong # of primary identifiers in "+desc(), 1, primaryIdentifiers.size()); + return attribute(primaryIdentifiers.iterator().next().getElementName()); + } + public ShadowAttributesAsserter assertHasPrimaryIdentifier() { Collection> primaryIdentifiers = ShadowUtil.getPrimaryIdentifiers(getShadow()); - assertFalse("No primary identifiers in "+desc(), primaryIdentifiers.isEmpty()); + assertFalse("No primary identifiers in "+desc(), CollectionUtils.isEmpty(primaryIdentifiers)); return this; } - + public ShadowAttributesAsserter assertNoPrimaryIdentifier() { Collection> primaryIdentifiers = ShadowUtil.getPrimaryIdentifiers(getShadow()); - assertTrue("Unexpected primary identifiers in "+desc()+": "+primaryIdentifiers, primaryIdentifiers.isEmpty()); + assertTrue("Unexpected primary identifiers in "+desc()+": "+primaryIdentifiers, CollectionUtils.isEmpty(primaryIdentifiers)); return this; } - + + public PrismPropertyAsserter> secondaryIdentifier() { + Collection> secondaryIdentifiers = ShadowUtil.getSecondaryIdentifiers(getShadow()); + assertFalse("No secondary identifier in "+desc(), CollectionUtils.isEmpty(secondaryIdentifiers)); + assertEquals("Wrong # of secondary identifiers in "+desc(), 1, secondaryIdentifiers.size()); + return attribute(secondaryIdentifiers.iterator().next().getElementName()); + } + public ShadowAttributesAsserter assertHasSecondaryIdentifier() { Collection> secondaryIdentifiers = ShadowUtil.getSecondaryIdentifiers(getShadow()); - assertFalse("No secondary identifiers in "+desc(), secondaryIdentifiers.isEmpty()); + assertFalse("No secondary identifiers in "+desc(), CollectionUtils.isEmpty(secondaryIdentifiers)); return this; } public ShadowAttributesAsserter assertNoSecondaryIdentifier() { Collection> secondaryIdentifiers = ShadowUtil.getSecondaryIdentifiers(getShadow()); - assertTrue("Unexpected secondary identifiers in "+desc()+": "+secondaryIdentifiers, secondaryIdentifiers.isEmpty()); + assertTrue("Unexpected secondary identifiers in "+desc()+": "+secondaryIdentifiers, CollectionUtils.isEmpty(secondaryIdentifiers)); return this; }