From 3c2e4417217f2f2c82bc6e3c353b99144ad254a3 Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Thu, 3 Sep 2015 12:57:55 +0200 Subject: [PATCH] Enhanced shadow integrity checker. Fixed bug in repo iterative search by paging. --- .../schema/constants/SchemaConstants.java | 1 + .../xml/ns/public/model/extension-3.xsd | 14 +- .../DefaultDuplicateShadowsResolver.java | 78 ++++ .../integrity/DuplicateShadowsResolver.java | 37 ++ .../DuplicateShadowsTreatmentInstruction.java | 55 +++ .../impl/integrity/ShadowCheckResult.java | 8 +- .../ShadowIntegrityCheckResultHandler.java | 410 +++++++++++++++--- .../ShadowIntegrityCheckTaskHandler.java | 6 +- .../model/impl/integrity/Statistics.java | 14 +- .../repo/sql/SqlRepositoryServiceImpl.java | 4 +- 10 files changed, 558 insertions(+), 69 deletions(-) create mode 100644 model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DefaultDuplicateShadowsResolver.java create mode 100644 model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DuplicateShadowsResolver.java create mode 100644 model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DuplicateShadowsTreatmentInstruction.java diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java index 5eabc124fed..2fd0b57ae17 100644 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java @@ -191,6 +191,7 @@ public abstract class SchemaConstants { public static final QName MODEL_EXTENSION_DIAGNOSE = new QName(NS_MODEL_EXTENSION, "diagnose"); public static final QName MODEL_EXTENSION_FIX = new QName(NS_MODEL_EXTENSION, "fix"); + public static final QName MODEL_EXTENSION_DUPLICATE_SHADOWS_RESOLVER = new QName(NS_MODEL_EXTENSION, "duplicateShadowsResolver"); public static final String NS_GUI = NS_MIDPOINT_PUBLIC + "/gui"; public static final String NS_GUI_CHANNEL = NS_GUI + "/channels-3"; diff --git a/infra/schema/src/main/resources/xml/ns/public/model/extension-3.xsd b/infra/schema/src/main/resources/xml/ns/public/model/extension-3.xsd index 04af6cf9c89..5db4807d987 100644 --- a/infra/schema/src/main/resources/xml/ns/public/model/extension-3.xsd +++ b/infra/schema/src/main/resources/xml/ns/public/model/extension-3.xsd @@ -224,7 +224,19 @@ - + + + + + Class that is used to resolve duplicate shadows in ShadowIntegrityCheck task. + If not specified, a default implementation is used. + + + 1 + + + + diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DefaultDuplicateShadowsResolver.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DefaultDuplicateShadowsResolver.java new file mode 100644 index 00000000000..810f324e897 --- /dev/null +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DefaultDuplicateShadowsResolver.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2010-2015 Evolveum + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.evolveum.midpoint.model.impl.integrity; + +import com.evolveum.midpoint.prism.PrismObject; +import com.evolveum.midpoint.schema.util.ObjectTypeUtil; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.SynchronizationSituationType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * @author Pavol Mederly + */ +public class DefaultDuplicateShadowsResolver implements DuplicateShadowsResolver { + + static final Trace LOGGER = TraceManager.getTrace(DefaultDuplicateShadowsResolver.class); + + @Override + public DuplicateShadowsTreatmentInstruction determineDuplicateShadowsTreatment(Collection> shadows) { + if (shadows.size() <= 1) { + return null; + } + LOGGER.trace("Determining duplicate shadows treatment: conflicting set with {} members", shadows.size()); + // prefer shadows with intent and linked ones + int bestScore = 0; + PrismObject bestShadow = null; + for (PrismObject shadow : shadows) { + ShadowType shadowType = shadow.asObjectable(); + int score = 0; + if (shadowType.getSynchronizationSituation() == SynchronizationSituationType.LINKED) { + score += 10; + } + List owners = (List) shadow.getUserData(ShadowIntegrityCheckResultHandler.KEY_OWNERS); + if (owners != null && !owners.isEmpty()) { + score += 10; + } + if (shadowType.getIntent() != null && !shadowType.getIntent().isEmpty()) { + score += 8; + } + if (shadow.getUserData(ShadowIntegrityCheckResultHandler.KEY_EXISTS_ON_RESOURCE) != null) { // filled in only if checkFetch is true + score += 50; + } + LOGGER.trace("Shadow {} has score of {}; best is {}", ObjectTypeUtil.toShortString(shadow), score, bestScore); + if (bestShadow == null || score > bestScore) { + bestScore = score; + bestShadow = shadow; + } + } + DuplicateShadowsTreatmentInstruction instruction = new DuplicateShadowsTreatmentInstruction(); + instruction.setShadowOidToReplaceDeletedOnes(bestShadow.getOid()); + instruction.setShadowsToDelete(new ArrayList>()); + for (PrismObject shadow : shadows) { + if (!shadow.getOid().equals(bestShadow.getOid())) { + instruction.getShadowsToDelete().add(shadow); + } + } + return instruction; + } +} diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DuplicateShadowsResolver.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DuplicateShadowsResolver.java new file mode 100644 index 00000000000..6fdded0acfc --- /dev/null +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DuplicateShadowsResolver.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2010-2015 Evolveum + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.evolveum.midpoint.model.impl.integrity; + +import com.evolveum.midpoint.prism.PrismObject; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType; + +import java.util.Collection; + +/** + * @author Pavol Mederly + */ +public interface DuplicateShadowsResolver { + + /** + * Takes a collection of duplicate shadows - i.e. shadows pointing to (presumably) one resource object, + * and returns a treatment instruction: a collection of shadows that have to be deleted + which OID to use in owner object as a replacement. + * + * @param shadows + * @return + */ + DuplicateShadowsTreatmentInstruction determineDuplicateShadowsTreatment(Collection> shadows); +} diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DuplicateShadowsTreatmentInstruction.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DuplicateShadowsTreatmentInstruction.java new file mode 100644 index 00000000000..7c3bf8314a5 --- /dev/null +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/DuplicateShadowsTreatmentInstruction.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2010-2015 Evolveum + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.evolveum.midpoint.model.impl.integrity; + +import com.evolveum.midpoint.prism.PrismObject; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType; + +import java.util.Collection; + +/** + * @author Pavol Mederly + */ +public class DuplicateShadowsTreatmentInstruction { + + private Collection> shadowsToDelete; + private String shadowOidToReplaceDeletedOnes; + + public Collection> getShadowsToDelete() { + return shadowsToDelete; + } + + public void setShadowsToDelete(Collection> shadowsToDelete) { + this.shadowsToDelete = shadowsToDelete; + } + + public String getShadowOidToReplaceDeletedOnes() { + return shadowOidToReplaceDeletedOnes; + } + + public void setShadowOidToReplaceDeletedOnes(String shadowOidToReplaceDeletedOnes) { + this.shadowOidToReplaceDeletedOnes = shadowOidToReplaceDeletedOnes; + } + + @Override + public String toString() { + return "DuplicateShadowsTreatmentInstruction{" + + "shadowsToDelete=" + shadowsToDelete + + ", shadowOidToReplaceDeletedOnes='" + shadowOidToReplaceDeletedOnes + '\'' + + '}'; + } +} diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowCheckResult.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowCheckResult.java index a11adb514cd..5aa89e4a325 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowCheckResult.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowCheckResult.java @@ -53,7 +53,9 @@ public ShadowCheckResult(PrismObject shadow) { } public ShadowCheckResult recordError(String problemCode, Exception e) { - problemCodes.add(problemCode); + if (problemCode != null) { + problemCodes.add(problemCode); + } LoggingUtils.logException(LOGGER, "{} - for shadow {} on resource {}", e, e.getMessage(), ObjectTypeUtil.toShortString(shadow), ObjectTypeUtil.toShortString(resource)); errors.add(e); @@ -61,7 +63,9 @@ public ShadowCheckResult recordError(String problemCode, Exception e) { } public ShadowCheckResult recordWarning(String problemCode, String message) { - problemCodes.add(problemCode); + if (problemCode != null) { + problemCodes.add(problemCode); + } LOGGER.warn("{} - for shadow {} on resource {}", message, ObjectTypeUtil.toShortString(shadow), ObjectTypeUtil.toShortString(resource)); warnings.add(message); diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowIntegrityCheckResultHandler.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowIntegrityCheckResultHandler.java index 6b4eca00c7d..1c6afa3269a 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowIntegrityCheckResultHandler.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowIntegrityCheckResultHandler.java @@ -18,19 +18,28 @@ import com.evolveum.midpoint.common.refinery.RefinedAttributeDefinition; import com.evolveum.midpoint.common.refinery.RefinedResourceSchema; +import com.evolveum.midpoint.model.impl.sync.SynchronizationService; import com.evolveum.midpoint.model.impl.util.AbstractSearchIterativeResultHandler; +import com.evolveum.midpoint.model.impl.util.Utils; import com.evolveum.midpoint.prism.Item; import com.evolveum.midpoint.prism.PrismContainer; import com.evolveum.midpoint.prism.PrismContext; import com.evolveum.midpoint.prism.PrismObject; import com.evolveum.midpoint.prism.PrismProperty; import com.evolveum.midpoint.prism.PrismPropertyValue; +import com.evolveum.midpoint.prism.PrismReferenceValue; +import com.evolveum.midpoint.prism.delta.ItemDelta; import com.evolveum.midpoint.prism.delta.PropertyDelta; +import com.evolveum.midpoint.prism.delta.ReferenceDelta; import com.evolveum.midpoint.prism.match.MatchingRule; import com.evolveum.midpoint.prism.match.MatchingRuleRegistry; import com.evolveum.midpoint.prism.path.ItemPath; +import com.evolveum.midpoint.prism.query.ObjectQuery; +import com.evolveum.midpoint.prism.query.RefFilter; import com.evolveum.midpoint.provisioning.api.ProvisioningService; import com.evolveum.midpoint.repo.api.RepositoryService; +import com.evolveum.midpoint.schema.GetOperationOptions; +import com.evolveum.midpoint.schema.SelectorOptions; import com.evolveum.midpoint.schema.constants.SchemaConstants; import com.evolveum.midpoint.schema.result.OperationResult; import com.evolveum.midpoint.schema.util.ObjectTypeUtil; @@ -40,24 +49,33 @@ import com.evolveum.midpoint.util.exception.CommonException; import com.evolveum.midpoint.util.exception.CommunicationException; import com.evolveum.midpoint.util.exception.ConfigurationException; +import com.evolveum.midpoint.util.exception.ExpressionEvaluationException; +import com.evolveum.midpoint.util.exception.ObjectAlreadyExistsException; import com.evolveum.midpoint.util.exception.ObjectNotFoundException; import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.util.exception.SecurityViolationException; import com.evolveum.midpoint.util.exception.SystemException; import com.evolveum.midpoint.util.logging.LoggingUtils; import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.xml.ns._public.common.common_3.FocusType; import com.evolveum.midpoint.xml.ns._public.common.common_3.LayerType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectSynchronizationType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowAttributesType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowKindType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.SynchronizationSituationType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.SystemConfigurationType; +import com.sun.jndi.toolkit.dir.SearchFilter; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import javax.xml.namespace.QName; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -72,24 +90,34 @@ public class ShadowIntegrityCheckResultHandler extends AbstractSearchIterativeRe static final Trace LOGGER = TraceManager.getTrace(ShadowIntegrityCheckResultHandler.class); private static final String CLASS_DOT = ShadowIntegrityCheckResultHandler.class.getName() + "."; + private static final String DEFAULT_DUPLICATE_SHADOWS_RESOLVER_CLASS_NAME = DefaultDuplicateShadowsResolver.class.getName(); + public static final String KEY_EXISTS_ON_RESOURCE = CLASS_DOT + "existsOnResource"; + public static final String KEY_OWNERS = CLASS_DOT + "owners"; private PrismContext prismContext; private ProvisioningService provisioningService; private MatchingRuleRegistry matchingRuleRegistry; private RepositoryService repositoryService; + private SynchronizationService synchronizationService; // derived from task extension diagnose/fix values at instantiation private boolean checkIntents; private boolean checkUniqueness; private boolean checkNormalization; + private boolean checkFetch; + private boolean checkOwners; private boolean fixIntents; private boolean fixUniqueness; private boolean fixNormalization; + private boolean dryRun; + public static final String INTENTS = "intents"; public static final String UNIQUENESS = "uniqueness"; public static final String NORMALIZATION = "normalization"; - public static final List KNOWN_KEYS = Arrays.asList(INTENTS, UNIQUENESS, NORMALIZATION); + public static final String OWNERS = "owners"; + public static final String FETCH = "fetch"; + public static final List KNOWN_KEYS = Arrays.asList(INTENTS, UNIQUENESS, NORMALIZATION, OWNERS, FETCH); // resource oid + kind -> ROCD // we silently assume that all intents for a given kind share a common attribute definition @@ -97,17 +125,25 @@ public class ShadowIntegrityCheckResultHandler extends AbstractSearchIterativeRe private Map> resources = new HashMap<>(); + private PrismObject configuration; + private Statistics statistics = new Statistics(); + DuplicateShadowsResolver duplicateShadowsResolver; + private Set duplicateShadowsDetected = new HashSet<>(); + private Set duplicateShadowsDeleted = new HashSet<>(); + public ShadowIntegrityCheckResultHandler(Task coordinatorTask, String taskOperationPrefix, String processShortName, String contextDesc, TaskManager taskManager, PrismContext prismContext, ProvisioningService provisioningService, MatchingRuleRegistry matchingRuleRegistry, - RepositoryService repositoryService) { + RepositoryService repositoryService, SynchronizationService synchronizationService, + OperationResult result) { super(coordinatorTask, taskOperationPrefix, processShortName, contextDesc, taskManager); this.prismContext = prismContext; this.provisioningService = provisioningService; this.matchingRuleRegistry = matchingRuleRegistry; this.repositoryService = repositoryService; + this.synchronizationService = synchronizationService; setStopOnError(false); setLogErrors(false); // we do log errors ourselves @@ -121,10 +157,14 @@ public ShadowIntegrityCheckResultHandler(Task coordinatorTask, String taskOperat checkIntents = true; checkUniqueness = true; checkNormalization = true; + checkOwners = true; + checkFetch = false; } else { checkIntents = contains(diagnosePrismProperty, INTENTS); checkUniqueness = contains(diagnosePrismProperty, UNIQUENESS); checkNormalization = contains(diagnosePrismProperty, NORMALIZATION); + checkOwners = contains(diagnosePrismProperty, OWNERS); + checkFetch = contains(diagnosePrismProperty, FETCH); checkProperty(diagnosePrismProperty); } PrismProperty fixPrismProperty = coordinatorTask.getExtensionProperty(SchemaConstants.MODEL_EXTENSION_FIX); @@ -139,6 +179,61 @@ public ShadowIntegrityCheckResultHandler(Task coordinatorTask, String taskOperat checkProperty(fixPrismProperty); } + if (fixIntents) { + checkIntents = true; + } + if (fixUniqueness) { + checkUniqueness = true; + } + if (fixNormalization) { + checkNormalization = true; + } + + if (fixUniqueness) { + PrismProperty duplicateShadowsResolverClass = coordinatorTask.getExtensionProperty(SchemaConstants.MODEL_EXTENSION_DUPLICATE_SHADOWS_RESOLVER); + String duplicateShadowsResolverClassName; + if (duplicateShadowsResolverClass != null) { + duplicateShadowsResolverClassName = duplicateShadowsResolverClass.getRealValue(); + } else { + duplicateShadowsResolverClassName = DEFAULT_DUPLICATE_SHADOWS_RESOLVER_CLASS_NAME; + } + try { + duplicateShadowsResolver = (DuplicateShadowsResolver) Class.forName(duplicateShadowsResolverClassName).newInstance(); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException | ClassCastException e) { + throw new SystemException("Couldn't instantiate duplicate shadows resolver " + duplicateShadowsResolverClassName); + } + } + + try { + configuration = Utils.getSystemConfiguration(repositoryService, result); + } catch (SchemaException e) { + throw new SystemException("Couldn't get system configuration", e); + } + + try { + dryRun = Utils.isDryRun(coordinatorTask); + } catch (SchemaException e) { + throw new SystemException("Couldn't get dryRun flag from task " + coordinatorTask); + } + + logConfiguration("Shadow integrity check is starting with the configuration:"); + } + + protected void logConfiguration(String state) { + LOGGER.info("{}\n" + + "- normalization diagnose={},\tfix={}\n" + + "- uniqueness diagnose={},\tfix={}\n" + + "- intents diagnose={},\tfix={}\n" + + "- owners diagnose={}\n" + + "- fetch diagnose={}\n\n" + + "dryRun = {}\n", + state, + checkNormalization, fixNormalization, + checkUniqueness, fixUniqueness, + checkIntents, fixIntents, + checkOwners, + checkFetch, + dryRun); } private void checkProperty(PrismProperty property) { @@ -235,10 +330,39 @@ private void checkShadow(ShadowCheckResult checkResult, PrismObject return; } + PrismObject fetchedShadow = null; + if (checkFetch) { + fetchedShadow = fetchShadow(checkResult, shadow, resource, workerTask, result); + if (fetchedShadow != null) { + shadow.setUserData(KEY_EXISTS_ON_RESOURCE, "true"); + } + } + + if (checkOwners) { + List owners = searchOwners(shadow, result); + if (owners != null) { + shadow.setUserData(KEY_OWNERS, owners); + if (owners.size() > 1) { + checkResult.recordError(Statistics.MULTIPLE_OWNERS, new SchemaException("Multiple owners: " + owners)); + } + } + + if (shadowType.getSynchronizationSituation() == SynchronizationSituationType.LINKED && (owners == null || owners.isEmpty())) { + checkResult.recordError(Statistics.LINKED_WITH_NO_OWNER, new SchemaException("Linked shadow with no owner")); + } + if (shadowType.getSynchronizationSituation() != SynchronizationSituationType.LINKED && owners != null && !owners.isEmpty()) { + checkResult.recordError(Statistics.NOT_LINKED_WITH_OWNER, new SchemaException("Shadow with an owner but not marked as linked (marked as " + + shadowType.getSynchronizationSituation() + ")")); + } + } + String intent = shadowType.getIntent(); - if (checkIntents && intent == null || intent.isEmpty()) { + if (checkIntents && (intent == null || intent.isEmpty())) { checkResult.recordWarning(Statistics.NO_INTENT_SPECIFIED, "None or empty intent"); } + if (fixIntents && (intent == null || intent.isEmpty())) { + doFixIntent(checkResult, fetchedShadow, shadow, resource, workerTask, result); + } Pair key = new ImmutablePair<>(resourceOid, kind); ObjectTypeContext context = contextMap.get(key); @@ -319,9 +443,82 @@ private void checkShadow(ShadowCheckResult checkResult, PrismObject } } + private List searchOwners(PrismObject shadow, OperationResult result) { + try { + PrismReferenceValue refValue = new PrismReferenceValue(shadow.getOid(), ShadowType.COMPLEX_TYPE); + RefFilter ownerFilter = RefFilter.createReferenceEqual(new ItemPath(FocusType.F_LINK_REF), FocusType.class, prismContext, refValue); + ObjectQuery ownerQuery = ObjectQuery.createObjectQuery(ownerFilter); + List owners = repositoryService.searchObjects(FocusType.class, ownerQuery, null, result); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("Owners for {}: {}", ObjectTypeUtil.toShortString(shadow), owners); + } + return owners; + } catch (SchemaException e) { + LoggingUtils.logUnexpectedException(LOGGER, "Couldn't create owners query for shadow {}", e, ObjectTypeUtil.toShortString(shadow)); + return null; + } + } + + private PrismObject fetchShadow(ShadowCheckResult checkResult, PrismObject shadow, PrismObject resource, Task task, OperationResult result) { + try { + PrismObject fullShadow = provisioningService.getObject(ShadowType.class, shadow.getOid(), + SelectorOptions.createCollection(GetOperationOptions.createDoNotDiscovery()), + task, result); + return fullShadow; + } catch (ObjectNotFoundException | CommunicationException | SchemaException | ConfigurationException | SecurityViolationException | RuntimeException e) { + checkResult.recordError(Statistics.CANNOT_FETCH_RESOURCE_OBJECT, new SystemException("The resource object couldn't be fetched", e)); + return null; + } + } + + private void doFixIntent(ShadowCheckResult checkResult, PrismObject fetchedShadow, PrismObject shadow, PrismObject resource, Task task, OperationResult result) { + PrismObject fullShadow; + + if (!checkFetch) { + fullShadow = fetchShadow(checkResult, shadow, resource, task, result); + } else { + fullShadow = fetchedShadow; + } + if (fullShadow == null) { + checkResult.recordError(Statistics.CANNOT_APPLY_FIX, new SystemException("Cannot fix missing intent, because the resource object couldn't be fetched")); + return; + } + + ObjectSynchronizationType synchronizationPolicy; + try { + synchronizationPolicy = synchronizationService.determineSynchronizationPolicy(resource.asObjectable(), fullShadow, configuration, task, result); + } catch (SchemaException|ObjectNotFoundException|ExpressionEvaluationException|RuntimeException e) { + checkResult.recordError(Statistics.CANNOT_APPLY_FIX, new SystemException("Couldn't prepare fix for missing intent, because the synchronization policy couldn't be determined", e)); + return; + } + if (synchronizationPolicy != null) { + if (synchronizationPolicy.getIntent() != null) { + PropertyDelta delta = PropertyDelta.createReplaceDelta(fullShadow.getDefinition(), ShadowType.F_INTENT, synchronizationPolicy.getIntent()); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("Intent fix delta (not executed now) = \n{}", delta.debugDump()); + } + checkResult.addFixDelta(delta, Statistics.NO_INTENT_SPECIFIED); + } else { + LOGGER.info("Synchronization policy does not contain intent: {}", synchronizationPolicy); + } + } else { + LOGGER.info("Intent couldn't be fixed, because no synchronization policy was found"); + } + } + private void applyFix(ShadowCheckResult checkResult, PrismObject shadow, Task workerTask, OperationResult result) throws CommonException { - LOGGER.info("Applying shadow fix:\n{}", DebugUtil.debugDump(checkResult.getFixDeltas())); - repositoryService.modifyObject(ShadowType.class, shadow.getOid(), checkResult.getFixDeltas(), result); + LOGGER.info("Applying shadow fix{}:\n{}", skippedForDryRun(), DebugUtil.debugDump(checkResult.getFixDeltas())); + if (!dryRun) { + repositoryService.modifyObject(ShadowType.class, shadow.getOid(), checkResult.getFixDeltas(), result); + } + } + + private String skippedForDryRun() { + if (dryRun) { + return " (skipped because of dry run)"; + } else { + return ""; + } } private void doCheckNormalization(ShadowCheckResult checkResult, RefinedAttributeDefinition identifier, String value, ObjectTypeContext context) { @@ -375,9 +572,10 @@ private void addIdentifierValue(ShadowCheckResult checkResult, ObjectTypeContext existingShadows.add(shadow); valueMap.put(identifierValue, existingShadows); } else { - checkResult.recordError(Statistics.DUPLICATE_SHADOWS, new SchemaException("Multiple shadows with the value of identifier " + - "attribute " + identifierName + " = " + identifierValue + ": existing one(s): " + shortDumpList(existingShadows) + - ", duplicate: " + ObjectTypeUtil.toShortString(shadow.asObjectable()))); + // duplicate shadows statistics are collected in a special way + duplicateShadowsDetected.add(shadow.getOid()); + LOGGER.error("Multiple shadows with the value of identifier attribute {} = {}: existing one(s): {}, duplicate: {}", + identifierName, identifierValue, shortDumpList(existingShadows), ObjectTypeUtil.toShortString(shadow.asObjectable())); existingShadows.add(shadow); } } @@ -401,67 +599,157 @@ public Statistics getStatistics() { return statistics; } - @Override - public void completeProcessing(OperationResult result) { - super.completeProcessing(result); + private String reportOrFixUniqueness(OperationResult result) { - LOGGER.info("Shadow integrity check finished.\n" + - " Shadows processed: {} ({} resources),\n" + - " Shadows with no problems: {}\n" + - " Shadows with warnings: {}\n" + - " Shadows with errors: {}\n" + - //" Shadows with uncompleted operations: {}\n" + - " Details:\n{}", - statistics.getShadows(), statistics.getResources(), - statistics.getShadows() - statistics.getShadowsWithErrors() - statistics.getShadowsWithWarnings(), - statistics.getShadowsWithWarnings(), statistics.getShadowsWithErrors(), - statistics.getDetailsFormatted()); + StringBuilder details = new StringBuilder(); + StringBuilder stat = new StringBuilder(); - if (checkUniqueness) { - StringBuilder sb = new StringBuilder(); - - for (Map.Entry, ObjectTypeContext> entry : contextMap.entrySet()) { - String resourceOid = entry.getKey().getLeft(); - ShadowKindType kind = entry.getKey().getRight(); - ObjectTypeContext ctx = entry.getValue(); - PrismObject resource = resources.get(resourceOid); - if (resource == null) { - LOGGER.error("No resource for {}", resourceOid); // should not happen - continue; - } - for (Map.Entry>>> idValEntry : ctx.getIdentifierValueMap().entrySet()) { - QName identifier = idValEntry.getKey(); - boolean first = true; - for (Map.Entry>> valListEntry : idValEntry.getValue().entrySet()) { - List> shadows = valListEntry.getValue(); - if (shadows.size() > 1) { - if (first) { - sb.append("Duplicates for ").append(ObjectTypeUtil.toShortString(resource)); - sb.append(", kind = ").append(kind); - sb.append(", identifier = ").append(identifier).append(":\n"); - first = false; - } - sb.append(" - value: ").append(valListEntry.getKey()).append(", shadows: ").append(shadows.size()).append("\n"); - for (PrismObject shadow : shadows) { - sb.append(" - ").append(ObjectTypeUtil.toShortString(shadow)); - sb.append("; sync situation = ").append(shadow.asObjectable().getSynchronizationSituation()).append("\n"); - PrismContainer attributesContainer = shadow.findContainer(ShadowType.F_ATTRIBUTES); - if (attributesContainer != null && !attributesContainer.isEmpty()) { - for (Item item : attributesContainer.getValue().getItems()) { - sb.append(" - ").append(item.getElementName().getLocalPart()).append(" = "); - sb.append(((PrismProperty) item).getRealValues()); - sb.append("\n"); - } - } + for (Map.Entry, ObjectTypeContext> entry : contextMap.entrySet()) { + String resourceOid = entry.getKey().getLeft(); + ShadowKindType kind = entry.getKey().getRight(); + ObjectTypeContext ctx = entry.getValue(); + PrismObject resource = resources.get(resourceOid); + if (resource == null) { + LOGGER.error("No resource for {}", resourceOid); // should not happen + continue; + } + for (Map.Entry>>> idValEntry : ctx.getIdentifierValueMap().entrySet()) { + QName identifier = idValEntry.getKey(); + boolean first = true; + for (Map.Entry>> valListEntry : idValEntry.getValue().entrySet()) { + List> shadows = valListEntry.getValue(); + if (shadows.size() <= 1) { + continue; + } + if (first) { + details.append("Duplicates for ").append(ObjectTypeUtil.toShortString(resource)); + details.append(", kind = ").append(kind); + details.append(", identifier = ").append(identifier).append(":\n"); + first = false; + } + details.append(" - value: ").append(valListEntry.getKey()).append(", shadows: ").append(shadows.size()).append("\n"); + List> shadowsToConsider = new ArrayList<>(); + for (PrismObject shadow : shadows) { + details.append(" - ").append(ObjectTypeUtil.toShortString(shadow)); + details.append("; sync situation = ").append(shadow.asObjectable().getSynchronizationSituation()).append("\n"); + PrismContainer attributesContainer = shadow.findContainer(ShadowType.F_ATTRIBUTES); + if (attributesContainer != null && !attributesContainer.isEmpty()) { + for (Item item : attributesContainer.getValue().getItems()) { + details.append(" - ").append(item.getElementName().getLocalPart()).append(" = "); + details.append(((PrismProperty) item).getRealValues()); + details.append("\n"); } } + if (duplicateShadowsDeleted.contains(shadow.getOid())) { + details.append(" (already deleted)\n"); + } else { + shadowsToConsider.add(shadow); + } + } + + if (fixUniqueness && shadowsToConsider.size() > 1) { + DuplicateShadowsTreatmentInstruction instruction = duplicateShadowsResolver.determineDuplicateShadowsTreatment(shadowsToConsider); + deleteShadows(instruction, details, result); } } } - if (sb.length() == 0) { - sb.append("No duplicates.\n"); + } + stat.append("Duplicate shadows detected: ").append(duplicateShadowsDetected.size()); + if (fixUniqueness) { + stat.append(", deleted: ").append(duplicateShadowsDeleted.size()); + // TODO report the duplicates that remain + } + + return stat.toString() + "\n" + details.toString(); + } + + // shadowsToDelete do not contain 'already deleted shadows' + private void deleteShadows(DuplicateShadowsTreatmentInstruction instruction, StringBuilder sb, OperationResult result) { + + LOGGER.trace("Going to delete shadows:\n{}", instruction); + if (instruction == null || instruction.getShadowsToDelete() == null) { + return; + } + Collection> shadowsToDelete = instruction.getShadowsToDelete(); + String shadowOidToReplaceDeleted = instruction.getShadowOidToReplaceDeletedOnes(); + + for (PrismObject shadowToDelete : shadowsToDelete) { + LOGGER.info("Deleting redundant shadow{} {}", skippedForDryRun(), ObjectTypeUtil.toShortString(shadowToDelete)); + sb.append(" --> deleted redundant shadow").append(skippedForDryRun()).append(" ").append(ObjectTypeUtil.toShortString(shadowToDelete)).append("\n"); + String oid = shadowToDelete.getOid(); + + List owners; + if (checkOwners) { + owners = (List) shadowToDelete.getUserData(KEY_OWNERS); + } else { + owners = searchOwners(shadowToDelete, result); + } + + if (!dryRun) { + try { + repositoryService.deleteObject(ShadowType.class, oid, result); + duplicateShadowsDeleted.add(oid); + } catch (ObjectNotFoundException e) { + // suspicious, but not a big deal + LoggingUtils.logExceptionAsWarning(LOGGER, "Shadow {} couldn't be deleted, because it does not exist anymore", e, ObjectTypeUtil.toShortString(shadowToDelete)); + continue; + } catch (RuntimeException e) { + LoggingUtils.logUnexpectedException(LOGGER, "Shadow {} couldn't be deleted because of an unexpected exception", e, ObjectTypeUtil.toShortString(shadowToDelete)); + continue; + } + } + + if (owners == null || owners.isEmpty()) { + continue; + } + + for (PrismObject owner : owners) { + List modifications = new ArrayList<>(2); + ReferenceDelta deleteDelta = ReferenceDelta.createModificationDelete(FocusType.F_LINK_REF, owner.getDefinition(), + new PrismReferenceValue(oid, ShadowType.COMPLEX_TYPE)); + modifications.add(deleteDelta); + if (shadowOidToReplaceDeleted != null) { + ReferenceDelta addDelta = ReferenceDelta.createModificationAdd(FocusType.F_LINK_REF, owner.getDefinition(), + new PrismReferenceValue(shadowOidToReplaceDeleted, ShadowType.COMPLEX_TYPE)); + modifications.add(addDelta); + } + LOGGER.info("Executing modify delta{} for owner {}:\n{}", skippedForDryRun(), ObjectTypeUtil.toShortString(owner), DebugUtil.debugDump(modifications)); + if (!dryRun) { + try { + repositoryService.modifyObject((Class) owner.getClass(), owner.getOid(), modifications, result); + } catch (ObjectNotFoundException | SchemaException | ObjectAlreadyExistsException | RuntimeException e) { + LoggingUtils.logUnexpectedException(LOGGER, "Focal object {} (owner of {}) couldn't be updated", e, ObjectTypeUtil.toShortString(owner), + ObjectTypeUtil.toShortString(shadowToDelete)); + } + } } - LOGGER.info("Uniqueness report:\n{}", sb.toString()); + + } + } + + @Override + public void completeProcessing(OperationResult result) { + super.completeProcessing(result); + + String uniquenessReport = null; + if (checkUniqueness) { + uniquenessReport = reportOrFixUniqueness(result); + } + + logConfiguration("Shadow integrity check finished. It was run with the configuration:"); + LOGGER.info("Results:\n" + + " Shadows processed: {} ({} resources),\n" + + " Shadows with no problems: {}\n" + + " Shadows with warnings: {}\n" + + " Shadows with errors: {}\n" + + " Details:\n{}", + statistics.getShadows(), statistics.getResources(), + statistics.getShadows() - statistics.getShadowsWithErrors() - statistics.getShadowsWithWarnings(), + statistics.getShadowsWithWarnings(), statistics.getShadowsWithErrors(), + statistics.getDetailsFormatted(dryRun)); + + if (uniquenessReport != null) { + LOGGER.info("Uniqueness report:\n{}", uniquenessReport); } } diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowIntegrityCheckTaskHandler.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowIntegrityCheckTaskHandler.java index 2cd8d689558..24bf490e920 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowIntegrityCheckTaskHandler.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/ShadowIntegrityCheckTaskHandler.java @@ -16,6 +16,7 @@ package com.evolveum.midpoint.model.impl.integrity; +import com.evolveum.midpoint.model.impl.sync.SynchronizationService; import com.evolveum.midpoint.model.impl.util.AbstractSearchIterativeTaskHandler; import com.evolveum.midpoint.prism.match.MatchingRuleRegistry; import com.evolveum.midpoint.prism.query.ObjectQuery; @@ -70,6 +71,9 @@ public class ShadowIntegrityCheckTaskHandler extends AbstractSearchIterativeTask @Autowired private MatchingRuleRegistry matchingRuleRegistry; + @Autowired + private SynchronizationService synchronizationService; + private static final Trace LOGGER = TraceManager.getTrace(ShadowIntegrityCheckTaskHandler.class); public ShadowIntegrityCheckTaskHandler() { @@ -86,7 +90,7 @@ private void initialize() { protected ShadowIntegrityCheckResultHandler createHandler(TaskRunResult runResult, Task coordinatorTask, OperationResult opResult) { return new ShadowIntegrityCheckResultHandler(coordinatorTask, ShadowIntegrityCheckTaskHandler.class.getName(), "check shadow integrity", "check shadow integrity", taskManager, prismContext, provisioningService, - matchingRuleRegistry, repositoryService); + matchingRuleRegistry, repositoryService, synchronizationService, opResult); } @Override diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/Statistics.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/Statistics.java index 507627d0a1e..9e823022190 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/Statistics.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/integrity/Statistics.java @@ -41,6 +41,10 @@ public class Statistics { public static final String NO_OBJECT_CLASS_REFINED_SCHEMA = "No object class refined schema"; public static final String OTHER_FAILURE = "Other failure"; public static final String CANNOT_APPLY_FIX = "Cannot apply fix"; + public static final String CANNOT_FETCH_RESOURCE_OBJECT = "Cannot fetch resource object"; + public static final String MULTIPLE_OWNERS = "Multiple owners"; + public static final String LINKED_WITH_NO_OWNER = "Linked shadow with no owner"; + public static final String NOT_LINKED_WITH_OWNER = "Not linked shadow with an owner"; private int resources; private int shadows; @@ -50,7 +54,6 @@ public class Statistics { private String[] codeList = { NON_NORMALIZED_IDENTIFIER_VALUE, - DUPLICATE_SHADOWS, NO_RESOURCE_OID, CANNOT_GET_RESOURCE, NO_KIND_SPECIFIED, @@ -58,6 +61,10 @@ public class Statistics { NO_RESOURCE_REFINED_SCHEMA, CANNOT_GET_REFINED_SCHEMA, NO_OBJECT_CLASS_REFINED_SCHEMA, + CANNOT_FETCH_RESOURCE_OBJECT, + MULTIPLE_OWNERS, + LINKED_WITH_NO_OWNER, + NOT_LINKED_WITH_OWNER, OTHER_FAILURE, CANNOT_APPLY_FIX }; @@ -158,7 +165,7 @@ public void registerProblemsFixes(List problemCodesFixed) { } } - public String getDetailsFormatted() { + public String getDetailsFormatted(boolean dryRun) { StringBuilder sb = new StringBuilder(); for (String code : codeList) { Counts counts = problemCount.get(code); @@ -172,6 +179,9 @@ public String getDetailsFormatted() { } if (fixable.contains(code)) { sb.append("; fixed "); + if (dryRun) { + sb.append("(if not run in dry-run mode) "); + } sb.append(counts.casesFixed).append(" cases"); if (counts.casesFixed > 0) { sb.append(" (").append(counts.shadowsFixed).append(" shadows)"); diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java index 3d22e782ae3..9c9758aa2e1 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java @@ -1851,7 +1851,7 @@ private void searchObjectsIterativeByPaging(Class type remaining = paging.getMaxSize() != null ? paging.getMaxSize() : countObjects(type, query, result) - offset; } - while (remaining > 0) { +main: while (remaining > 0) { paging.setOffset(offset); paging.setMaxSize(remaining < batchSize ? remaining : batchSize); @@ -1859,7 +1859,7 @@ private void searchObjectsIterativeByPaging(Class type for (PrismObject object : objects) { if (!handler.handle(object, result)) { - break; + break main; } }