Fix updating primaryIdentifierValue on deletion

As soon as the account deletion is attempted, the primaryIdentifierValue
in its shadow is now erased. And, after the (retryable) deletion
operation is determined to be (definitely) failed, the value is
attempted to be restored.

This should fix failing provisioning-impl tests.

Related to MID-8069.

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadows/DeleteHelper.java

@@ -121,8 +121,8 @@ PrismObject<ShadowType> deleteShadowAttempt(ProvisioningContext ctx,
 
         shadowCaretaker.applyAttributesDefinition(ctx, opState.getRepoShadow());
 
-        PendingOperationType duplicateOperation = shadowManager.checkAndRecordPendingDeleteOperationBeforeExecution(ctx,
-                opState, result);
+        PendingOperationType duplicateOperation =
+                shadowManager.checkAndRecordPendingDeleteOperationBeforeExecution(ctx, opState, result);
         if (duplicateOperation != null) {
             result.setInProgress();
             return opState.getRepoShadow();
@@ -200,8 +200,8 @@ private OperationResultStatus deleteShadowDirectly(ProvisioningContext ctx, Prov
         try {
             ctx.checkNotInMaintenance();
 
-            AsynchronousOperationResult asyncReturnValue = resourceObjectConverter
-                    .deleteResourceObject(ctx, repoShadow, scripts, connOptions, result);
+            AsynchronousOperationResult asyncReturnValue =
+                    resourceObjectConverter.deleteResourceObject(ctx, repoShadow, scripts, connOptions, result);
             opState.processAsyncResult(asyncReturnValue);
 
             resourceManager.modifyResourceAvailabilityStatus(ctx.getResourceOid(), AvailabilityStatusType.UP,

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadows/GetHelper.java

@@ -171,10 +171,9 @@ class GetHelper {
         if (shouldRefreshOnRead(resource, rootOptions)) {
             LOGGER.trace("Refreshing {} before reading", repoShadow);
             ProvisioningOperationOptions refreshOpts = toProvisioningOperationOptions(rootOptions);
-            RefreshShadowOperation refreshShadowOperation = refreshHelper.refreshShadow(repoShadow, refreshOpts, task, result);
-            if (refreshShadowOperation != null) {
-                repoShadow = refreshShadowOperation.getRefreshedShadow();
-            }
+            repoShadow = refreshHelper
+                    .refreshShadow(repoShadow, refreshOpts, task, result)
+                    .getRefreshedShadow();
             LOGGER.trace("Refreshed repository shadow:\n{}", DebugUtil.debugDumpLazily(repoShadow, 1));
         }
         if (repoShadow == null) {

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadows/ModifyHelper.java

@@ -18,6 +18,7 @@
 import com.evolveum.midpoint.provisioning.impl.resourceobjects.ResourceObjectConverter;
 
 import org.apache.commons.lang3.Validate;
+import org.jetbrains.annotations.Contract;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -164,13 +165,12 @@ String modifyShadowAttempt(ProvisioningContext ctx,
             if (shouldExecuteResourceOperationDirectly(ctx)) {
                 LOGGER.trace("MODIFY {}: resource modification, execution starting\n{}", repoShadow, DebugUtil.debugDumpLazily(modifications));
 
-                RefreshShadowOperation refreshShadowOperation = null;
+                RefreshShadowOperation refreshShadowOperation;
                 if (!inRefresh && Util.shouldRefresh(repoShadow)) {
                     refreshShadowOperation = refreshHelper.refreshShadow(repoShadow, options, task, parentResult);
-                }
-
-                if (refreshShadowOperation != null) {
                     repoShadow = refreshShadowOperation.getRefreshedShadow();
+                } else {
+                    refreshShadowOperation = null;
                 }
 
                 if (repoShadow == null) {
@@ -231,6 +231,7 @@ String modifyShadowAttempt(ProvisioningContext ctx,
         return repoShadow.getOid();
     }
 
+    @Contract("null -> true")
     private boolean shouldExecuteModify(RefreshShadowOperation refreshShadowOperation) {
         if (refreshShadowOperation == null) {
             LOGGER.trace("Nothing refreshed, modify can continue.");

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadows/RefreshHelper.java

@@ -9,6 +9,7 @@
 
 import static com.evolveum.midpoint.provisioning.impl.shadows.Util.createSuccessOperationDescription;
 import static com.evolveum.midpoint.provisioning.impl.shadows.Util.needsRetry;
+import static com.evolveum.midpoint.util.MiscUtil.emptyIfNull;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -20,7 +21,7 @@
 
 import com.evolveum.midpoint.schema.processor.ResourceAttributeDefinition;
 
-import org.jetbrains.annotations.Nullable;
+import org.jetbrains.annotations.NotNull;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -88,9 +89,8 @@ class RefreshHelper {
     @Autowired private DeleteHelper deleteHelper;
     @Autowired private DefinitionsHelper definitionsHelper;
 
-    @Nullable
-    public RefreshShadowOperation refreshShadow(PrismObject<ShadowType> repoShadow, ProvisioningOperationOptions options,
-            Task task, OperationResult result)
+    public @NotNull RefreshShadowOperation refreshShadow(
+            PrismObject<ShadowType> repoShadow, ProvisioningOperationOptions options, Task task, OperationResult result)
             throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException,
             ExpressionEvaluationException, EncryptionException {
 
@@ -99,7 +99,11 @@ public RefreshShadowOperation refreshShadow(PrismObject<ShadowType> repoShadow,
         ctx.assertDefinition();
         shadowCaretaker.applyAttributesDefinition(ctx, repoShadow);
 
-        repoShadow = shadowManager.refreshProvisioningIndexes(ctx, repoShadow, task, result);
+        try {
+            shadowManager.refreshProvisioningIndexes(ctx, repoShadow, true, result);
+        } catch (ObjectAlreadyExistsException e) {
+            throw SystemException.unexpected(e, "when refreshing provisioning indexes");
+        }
 
         RefreshShadowOperation refreshShadowOperation = refreshShadowPendingOperations(ctx, repoShadow, options, task, result);
 
@@ -109,12 +113,21 @@ public RefreshShadowOperation refreshShadow(PrismObject<ShadowType> repoShadow,
         if (shadowAfterCleanup == null) {
             refreshShadowOperation.setRefreshedShadow(null);
         }
+
+        updateProvisioningIndexesAfterDeletion(ctx, refreshShadowOperation, result);
+
         return refreshShadowOperation;
     }
 
-    private RefreshShadowOperation refreshShadowPendingOperations(ProvisioningContext ctx, PrismObject<ShadowType> repoShadow,
-            ProvisioningOperationOptions options, Task task, OperationResult parentResult) throws ObjectNotFoundException,
-            SchemaException, CommunicationException, ConfigurationException, ExpressionEvaluationException, EncryptionException {
+    private @NotNull RefreshShadowOperation refreshShadowPendingOperations(
+            ProvisioningContext ctx,
+            PrismObject<ShadowType> repoShadow,
+            ProvisioningOperationOptions options,
+            Task task,
+            OperationResult result)
+            throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException,
+            ExpressionEvaluationException {
+
         ShadowType shadowType = repoShadow.asObjectable();
         List<PendingOperationType> pendingOperations = shadowType.getPendingOperation();
         boolean isDead = ShadowUtil.isDead(shadowType);
@@ -137,9 +150,9 @@ private RefreshShadowOperation refreshShadowPendingOperations(ProvisioningContex
         ctx.assertDefinition();
         List<PendingOperationType> sortedOperations = shadowCaretaker.sortPendingOperations(shadowType.getPendingOperation());
 
-        refreshShadowAsyncStatus(ctx, repoShadow, sortedOperations, task, parentResult);
+        refreshShadowAsyncStatus(ctx, repoShadow, sortedOperations, task, result);
 
-        return refreshShadowRetryOperations(ctx, repoShadow, sortedOperations, options, task, parentResult);
+        return refreshShadowRetryOperations(ctx, repoShadow, sortedOperations, options, task, result);
     }
 
     /**
@@ -359,9 +372,15 @@ private boolean needsRefresh(PendingOperationType pendingOperation) {
         }
     }
 
-    private RefreshShadowOperation refreshShadowRetryOperations(ProvisioningContext ctx,
-            PrismObject<ShadowType> repoShadow, List<PendingOperationType> sortedOperations,
-            ProvisioningOperationOptions options, Task task, OperationResult parentResult) throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, ExpressionEvaluationException, EncryptionException {
+    private @NotNull RefreshShadowOperation refreshShadowRetryOperations(
+            ProvisioningContext ctx,
+            PrismObject<ShadowType> repoShadow,
+            List<PendingOperationType> sortedOperations,
+            ProvisioningOperationOptions options,
+            Task task,
+            OperationResult parentResult)
+            throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException,
+            ExpressionEvaluationException{
         ShadowType shadowType = repoShadow.asObjectable();
         OperationResult retryResult = new OperationResult(OP_REFRESH_RETRY);
         if (ShadowUtil.isDead(shadowType)) {
@@ -576,4 +595,33 @@ private void expirePendingOperations(ProvisioningContext ctx, PrismObject<Shadow
         }
     }
 
+    /**
+     * When a deletion is determined to be failed, we try to restore the `primaryIdentifierValue` index.
+     * (We may be unsuccessful if there was another shadow created in the meanwhile.)
+     *
+     * This method assumes that the pending operations have been already updated with the result of retried operations.
+     *
+     * (For simplicity and robustness, we just refresh provisioning indexes. It should be efficient enough.)
+     */
+    private void updateProvisioningIndexesAfterDeletion(
+            ProvisioningContext ctx, RefreshShadowOperation refreshShadowOperation, OperationResult result)
+            throws SchemaException, ObjectNotFoundException {
+        PrismObject<ShadowType> shadow = refreshShadowOperation.getRefreshedShadow();
+        if (shadow == null) {
+            LOGGER.trace("updateProvisioningIndexesAfterDeletion: no shadow");
+            return;
+        }
+        if (emptyIfNull(refreshShadowOperation.getExecutedDeltas()).stream()
+                .noneMatch(d -> ObjectDelta.isDelete(d.getObjectDelta()))) {
+            LOGGER.trace("updateProvisioningIndexesAfterDeletion: no DELETE delta found");
+            return;
+        }
+        try {
+            shadowManager.refreshProvisioningIndexes(ctx, shadow, false, result);
+        } catch (ObjectAlreadyExistsException e) {
+            LOGGER.debug("Couldn't set `primaryIdentifierValue` for {} - probably a new one was created in the meanwhile. "
+                    + "Marking this one as dead.", shadow, e);
+            shadowManager.markShadowTombstone(shadow, ctx.getTask(), result);
+        }
+    }
 }

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadows/manager/PendingOperationsHelper.java

@@ -20,6 +20,7 @@
 import com.evolveum.midpoint.schema.result.AsynchronousOperationResult;
 import com.evolveum.midpoint.schema.result.AsynchronousOperationReturnValue;
 import com.evolveum.midpoint.schema.result.OperationResultStatus;
+import com.evolveum.midpoint.util.MiscUtil;
 import com.evolveum.midpoint.util.annotation.Experimental;
 
 import com.evolveum.midpoint.util.exception.SchemaException;
@@ -91,16 +92,12 @@ void collectPendingOperationUpdates(Collection<ItemDelta<?, ?>> shadowChanges,
             OperationResultStatus implicitStatus,
             XMLGregorianCalendar now) {
 
-        PrismContainerDefinition<PendingOperationType> containerDefinition = opState.getRepoShadow().getDefinition().findContainerDefinition(ShadowType.F_PENDING_OPERATION);
+        PrismContainerDefinition<PendingOperationType> containerDefinition =
+                opState.getRepoShadow().getDefinition().findContainerDefinition(ShadowType.F_PENDING_OPERATION);
 
-        OperationResultStatus opStateResultStatus = opState.getResultStatus();
-        if (opStateResultStatus == null) {
-            opStateResultStatus = implicitStatus;
-        }
-        OperationResultStatusType opStateResultStatusType = null;
-        if (opStateResultStatus != null) {
-            opStateResultStatusType = opStateResultStatus.createStatusType();
-        }
+        OperationResultStatus opStateResultStatus = MiscUtil.getFirstNonNull(opState.getResultStatus(), implicitStatus);
+        OperationResultStatusType opStateResultStatusType =
+                opStateResultStatus != null ? opStateResultStatus.createStatusType() : null;
         String asynchronousOperationReference = opState.getAsynchronousOperationReference();
 
         for (PendingOperationType pendingOperation : opState.getPendingOperations()) {
@@ -133,7 +130,7 @@ void collectPendingOperationUpdates(Collection<ItemDelta<?, ?>> shadowChanges,
 
                 if (pendingOperation.getRequestTimestamp() == null) {
                     // This is mostly failsafe. We do not want operations without timestamps. Those would be quite difficult to cleanup.
-                    // Therefore unprecise timestamp is better than no timestamp.
+                    // Therefore imprecise timestamp is better than no timestamp.
                     PropertyDelta<XMLGregorianCalendar> timestampDelta = createPendingOperationDelta(containerDefinition, containerPath,
                             PendingOperationType.F_REQUEST_TIMESTAMP, now);
                     shadowChanges.add(timestampDelta);

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/shadows/manager/ShadowManager.java

@@ -12,7 +12,6 @@
 import javax.xml.datatype.XMLGregorianCalendar;
 import javax.xml.namespace.QName;
 
-import com.evolveum.midpoint.schema.processor.ResourceObjectTypeDefinition;
 import com.evolveum.midpoint.schema.processor.ResourceObjectDefinition;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowLifecycleStateType;
 
@@ -280,9 +279,13 @@ public <T> T determinePrimaryIdentifierValue(ProvisioningContext ctx, PrismObjec
         return helper.determinePrimaryIdentifierValue(ctx, shadow);
     }
 
-    public PrismObject<ShadowType> refreshProvisioningIndexes(ProvisioningContext ctx, PrismObject<ShadowType> repoShadow,
-            Task task, OperationResult result) throws ObjectNotFoundException, SchemaException {
-        return shadowUpdater.refreshProvisioningIndexes(ctx, repoShadow, task, result);
+    /**
+     * @throws ObjectAlreadyExistsException Only if `resolveConflicts` is `false`
+     */
+    public void refreshProvisioningIndexes(
+            ProvisioningContext ctx, PrismObject<ShadowType> repoShadow, boolean resolveConflicts, OperationResult result)
+            throws ObjectNotFoundException, SchemaException, ObjectAlreadyExistsException {
+        shadowUpdater.refreshProvisioningIndexes(ctx, repoShadow, resolveConflicts, result);
     }
 
     public void recordModifyResult(ProvisioningContext ctx, PrismObject<ShadowType> oldRepoShadow,