Consistency update: the devil is in the details (MID-3891)

infra/common/src/main/java/com/evolveum/midpoint/common/SynchronizationUtils.java

@@ -197,25 +197,25 @@ public static PropertyDelta<XMLGregorianCalendar> createSynchronizationTimestamp
 		return syncSituationDelta;
 	}
 
-	public static List<PropertyDelta<?>> createSynchronizationSituationAndDescriptionDelta(PrismObject object,
+	public static List<PropertyDelta<?>> createSynchronizationSituationAndDescriptionDelta(PrismObject<ShadowType> currentObject,
 			SynchronizationSituationType situation, String sourceChannel, boolean full) {
 		XMLGregorianCalendar timestamp = XmlTypeConverter
 				.createXMLGregorianCalendar(System.currentTimeMillis());
 
-		List<PropertyDelta<?>> delta = createSynchronizationSituationDescriptionDelta(object, situation,
+		List<PropertyDelta<?>> delta = createSynchronizationSituationDescriptionDelta(currentObject, situation,
 				timestamp, sourceChannel, full);
 
-		PropertyDelta<XMLGregorianCalendar> timestampDelta = createSynchronizationTimestampDelta(object,
+		PropertyDelta<XMLGregorianCalendar> timestampDelta = createSynchronizationTimestampDelta(currentObject,
 				ShadowType.F_SYNCHRONIZATION_TIMESTAMP, timestamp);
 		delta.add(timestampDelta);
 
 		if (full) {
-			timestampDelta = createSynchronizationTimestampDelta(object,
+			timestampDelta = createSynchronizationTimestampDelta(currentObject,
 					ShadowType.F_FULL_SYNCHRONIZATION_TIMESTAMP, timestamp);
 			delta.add(timestampDelta);
 		}
 
-		PropertyDelta<SynchronizationSituationType> syncSituationDelta = createSynchronizationSituationDelta(object, situation);
+		PropertyDelta<SynchronizationSituationType> syncSituationDelta = createSynchronizationSituationDelta(currentObject, situation);
 
 		if (syncSituationDelta != null) {
 			delta.add(syncSituationDelta);

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/ChangeExecutor.java

@@ -632,7 +632,7 @@ private <O extends ObjectType, F extends FocusType> void updateLinks(
 									// Already linked, nothing to do, only be sure, the
 									// situation is set with the good value
 									LOGGER.trace("Updating situation in already linked shadow.");
-									updateSituationInShadow(task, SynchronizationSituationType.LINKED, null, focusObjectContext,
+									updateSituationInShadow(task, SynchronizationSituationType.LINKED, focusObjectContext,
 											projCtx, result);
 									return;
 								}
@@ -642,7 +642,7 @@ private <O extends ObjectType, F extends FocusType> void updateLinks(
 						linkShadow(focusContext.getOid(), projOid, focusObjectContext, projCtx, task, result);
 						// be sure, that the situation is set correctly
 						LOGGER.trace("Updating situation after shadow was linked.");
-						updateSituationInShadow(task, SynchronizationSituationType.LINKED, null, focusObjectContext, projCtx,
+						updateSituationInShadow(task, SynchronizationSituationType.LINKED,  focusObjectContext, projCtx,
 								result);
 		} else {
 			// Link should NOT exist
@@ -665,26 +665,11 @@ private <O extends ObjectType, F extends FocusType> void updateLinks(
 				}
 			}
 
-			if (projCtx.isDelete() || projCtx.isThombstone()) {
-				LOGGER.trace("Resource object {} deleted, updating also situation in shadow.", projOid);
-				// HACK HACK?
-				try {
-					updateSituationInShadow(task, SynchronizationSituationType.DELETED, true, focusObjectContext,
-							projCtx, result);
-				} catch (ObjectNotFoundException e) {
-					// HACK HACK?
-					LOGGER.trace(
-							"Resource object {} is gone, cannot update situation in shadow (this is probably harmless).",
-							projOid);
-					result.getLastSubresult().setStatus(OperationResultStatus.HANDLED_ERROR);
-				}
-			} else {
-				// This should NOT be UNLINKED. We just do not know the
-				// situation here. Reflect that in the shadow.
-				LOGGER.trace("Resource object {} unlinked from the user, updating also situation in shadow.",
-						projOid);
-				updateSituationInShadow(task, null, null, focusObjectContext, projCtx, result);
-			}
+			// This should NOT be UNLINKED. We just do not know the
+			// situation here. Reflect that in the shadow.
+			LOGGER.trace("Resource object {} unlinked from the user, updating also situation in shadow.",
+					projOid);
+			updateSituationInShadow(task, null, focusObjectContext, projCtx, result);
 			// Not linked, that's OK
 		}
 	}
@@ -803,7 +788,7 @@ private <F extends ObjectType> void unlinkShadow(String focusOid, PrismReference
 	}
 
 	private <F extends ObjectType> void updateSituationInShadow(Task task,
-			SynchronizationSituationType situation, Boolean dead, LensFocusContext<F> focusContext,
+			SynchronizationSituationType situation, LensFocusContext<F> focusContext,
 			LensProjectionContext projectionCtx, OperationResult parentResult)
 					throws ObjectNotFoundException, SchemaException {
 
@@ -826,11 +811,6 @@ private <F extends ObjectType> void updateSituationInShadow(Task task,
 		List<PropertyDelta<?>> syncSituationDeltas = SynchronizationUtils
 				.createSynchronizationSituationAndDescriptionDelta(account, situation, task.getChannel(),
 						projectionCtx.hasFullShadow());
-
-		if (dead != null) {
-			PropertyDelta<Boolean> deadDelta = PropertyDelta.createModificationReplaceProperty(ShadowType.F_DEAD, account.getDefinition(), dead);
-			syncSituationDeltas.add(deadDelta);
-		}
 
 		try {
 			Utils.setRequestee(task, focusContext);

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/TestMultiResource.java

@@ -534,9 +534,8 @@ public void test223JackKillDefaultDummyAccounAndRecompute() throws Exception {
 		displayThen(TEST_NAME);
 		assertSuccess(result);
 
-        PrismObject<UserType> userJack = getUser(USER_JACK_OID);
-        display("user after", userJack);
-        assertLinks(userJack, 2);
+		assertUserAfter(USER_JACK_OID)
+			.assertLinks(2);
 
         display("dummy resource after", getDummyResource());
 
@@ -572,12 +571,29 @@ public void test224JackKillBeigeAccounAndRecompute() throws Exception {
 
 		// THEN
 		displayThen(TEST_NAME);
-		result.computeStatus();
-		display("Result", result);
-        TestUtil.assertSuccess(result);
+		assertSuccess(result);
 
-        PrismObject<UserType> userJack = getUser(USER_JACK_OID);
-        assertLinks(userJack, 2);
+		assertUserAfter(USER_JACK_OID)
+			.displayWithProjections()
+			.links()
+				.assertLinks(3)
+				.by()
+					.resourceOid(RESOURCE_DUMMY_BEIGE_OID)
+					.dead(true)
+				.find()
+					.end()
+				.by()
+					.resourceOid(RESOURCE_DUMMY_BEIGE_OID)
+					.dead(false)
+				.find()
+					.end()
+				.by()
+					.resourceOid(RESOURCE_DUMMY_OID)
+				.find()
+					.target()
+						.assertLife()
+						.end()
+					.end();
 
         display("beige dummy resource after", getDummyResource(RESOURCE_DUMMY_BEIGE_NAME));
 

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractDirectManualResourceTest.java

@@ -1180,7 +1180,7 @@ public void test302RecomputeWill() throws Exception {
 	}
 
 	/**
-	 * Case is closed. The operation is complete.
+	 * Case is closed. The unassign operation is complete.
 	 * However, in the semi-manual case this gets really interesting.
 	 * We have Schroedinger's shadow here.  deleted account, ticket closed, account is deleted
 	 * by administrator in the target system. But the account is still in the backing store (CSV)
@@ -1346,7 +1346,7 @@ public void test330UpdateBackingStoreAndRecomputeWill() throws Exception {
 				.end();
 		assertUnassignedShadow(shadowRepoAsserter, true, null);
 
-		ShadowAsserter<Void> shadowModelAsserter = assertModelShadow(accountWillOid)
+		ShadowAsserter<Void> shadowModelAsserter = assertModelShadowNoFetch(accountWillOid)
 			.assertName(USER_WILL_NAME)
 			.assertKind(ShadowKindType.ACCOUNT)
 			.pendingOperations()
@@ -1360,12 +1360,14 @@ public void test330UpdateBackingStoreAndRecomputeWill() throws Exception {
 		assertUnassignedShadow(shadowModelAsserter, true, ActivationStatusType.DISABLED);
 		// Do NOT assert password here. There is no password even for semi-manual case as the shadow is dead and account gone.
 
-		PrismObject<ShadowType> shadowModelFuture = modelService.getObject(ShadowType.class,
-				accountWillOid,
-				SelectorOptions.createCollection(GetOperationOptions.createPointInTimeType(PointInTimeType.FUTURE)),
-				task, result);
-		display("Model shadow (future)", shadowModelFuture);
-		assertWillUnassignedFuture(shadowModelFuture, false);
+		assertModelShadow(accountWillOid)
+			.assertTombstone();
+
+		assertModelShadowFuture(accountWillOid)
+			.assertTombstone();
+
+		assertModelShadowFutureNoFetch(accountWillOid)
+			.assertTombstone();
 
 		assertCase(willLastCaseOid, SchemaConstants.CASE_STATE_CLOSED);
 
@@ -1738,7 +1740,7 @@ public void test515CloseCasesAndReconcileWill() throws Exception {
 				SelectorOptions.createCollection(GetOperationOptions.createPointInTimeType(PointInTimeType.FUTURE)),
 				task, result);
 		display("Model shadow (future)", shadowModelFuture);
-		assertWillUnassignedFuture(shadowModelFuture, true);
+		assertWillUnassignedFuture(shadowModelFuture, false);
 
 		assertCase(willLastCaseOid, SchemaConstants.CASE_STATE_CLOSED);
 		assertCase(willSecondLastCaseOid, SchemaConstants.CASE_STATE_CLOSED);
@@ -1988,7 +1990,7 @@ public void test524TwoHoursForRoleTwo() throws Exception {
 	 */
 	@Test
 	public void test525CloseCasesAndRecomputeWill() throws Exception {
-		final String TEST_NAME = "test515CloseCasesAndRecomputeWill";
+		final String TEST_NAME = "test525CloseCasesAndRecomputeWill";
 		displayTestTitle(TEST_NAME);
 		// GIVEN
 		Task task = createTask(TEST_NAME);

model/model-test/src/main/java/com/evolveum/midpoint/model/test/AbstractModelIntegrationTest.java

@@ -5291,6 +5291,14 @@ protected ShadowAsserter<Void> assertModelShadow(String oid) throws ObjectNotFou
 		return asserter;
 	}
 
+	protected ShadowAsserter<Void> assertModelShadowNoFetch(String oid) throws ObjectNotFoundException, SchemaException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException {
+		PrismObject<ShadowType> repoShadow = getShadowModelNoFetch(oid);
+		ShadowAsserter<Void> asserter = ShadowAsserter.forShadow(repoShadow, "model(noFetch)");
+		asserter
+			.display();
+		return asserter;
+	}
+
 	protected ShadowAsserter<Void> assertModelShadowFuture(String oid) throws ObjectNotFoundException, SchemaException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException {
 		PrismObject<ShadowType> repoShadow = getShadowModelFuture(oid);
 		ShadowAsserter<Void> asserter = ShadowAsserter.forShadow(repoShadow, "model(future)");
@@ -5299,6 +5307,16 @@ protected ShadowAsserter<Void> assertModelShadowFuture(String oid) throws Object
 		return asserter;
 	}
 
+	protected ShadowAsserter<Void> assertModelShadowFutureNoFetch(String oid) throws ObjectNotFoundException, SchemaException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException {
+		GetOperationOptions options = GetOperationOptions.createPointInTimeType(PointInTimeType.FUTURE);
+		options.setNoFetch(true);
+		PrismObject<ShadowType> repoShadow = getShadowModel(oid, options, true);
+		ShadowAsserter<Void> asserter = ShadowAsserter.forShadow(repoShadow, "model(future,noFetch)");
+		asserter
+			.display();
+		return asserter;
+	}
+
 	protected void assertNoModelShadow(String oid) throws SchemaException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException {
 		Task task = createTask("assertNoModelShadow");
 		OperationResult result = task.getResult();
@@ -5310,5 +5328,18 @@ protected void assertNoModelShadow(String oid) throws SchemaException, SecurityV
 			assertFailure(result);
 		}
 	}
+
+	protected void assertNoModelShadowFuture(String oid) throws SchemaException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException {
+		Task task = createTask("assertNoModelShadowFuture");
+		OperationResult result = task.getResult();
+		Collection<SelectorOptions<GetOperationOptions>> options = SelectorOptions.createCollection(GetOperationOptions.createPointInTimeType(PointInTimeType.FUTURE));
+		try {
+			PrismObject<ShadowType> shadow = modelService.getObject(ShadowType.class, oid, options, task, result);
+			fail("Expected that future shadow "+oid+" will not be in the model. But it was: "+shadow);
+		} catch (ObjectNotFoundException e) {
+			// Expected
+			assertFailure(result);
+		}
+	}
 
 }

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ProvisioningServiceImpl.java

@@ -257,7 +257,7 @@ public <T extends ObjectType> PrismObject<T> getObject(Class<T> type, String oid
 			}
 		}
 
-		result.computeStatus();
+		result.computeStatusIfUnknown();
 		if (!GetOperationOptions.isRaw(rootOptions)) {
 			resultingObject = resultingObject.cloneIfImmutable();
 			resultingObject.asObjectable().setFetchResult(result.createOperationResultType());

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java

@@ -2605,7 +2605,9 @@ private PrismObject<ShadowType> completeShadow(ProvisioningContext ctx,
 		ProvisioningUtil.setProtectedFlag(ctx, resultShadow, matchingRuleRegistry);
 
 		// exists, dead
-		resultShadowType.setExists(resourceShadowType.isExists());
+		// This may seem strange, but always take exists and dead flags from the repository.
+		// Repository is wiser in this case. It may seem that the shadow exists if it is returned
+		// by the resource. But that may be just a quantum illusion (gestation and corpse shadow states).
 
 		// Activation
 		ActivationType resultActivationType = resultShadowType.getActivation();

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectNotFoundHandler.java

@@ -92,6 +92,15 @@ public PrismObject<ShadowType> handleGetError(ProvisioningContext ctx,
 			discoverDeletedShadow(ctx, repositoryShadow, cause, task, parentResult);
 		}
 
+		if (repositoryShadow != null) {
+			// We always want to return repository shadow it such shadow is available.
+			// The shadow may be dead, or it may be marked as "not exists", but we want
+			// to return something if shadow exists in the repo. Otherwise model may
+			// unlink the shadow or otherwise "forget" about it.
+			LOGGER.debug("Shadow {} not found on the resource. However still have it in the repository. Therefore returning repository version.", repositoryShadow);
+			parentResult.setStatus(OperationResultStatus.HANDLED_ERROR);
+			return repositoryShadow;
+		}
 		return super.handleGetError(ctx, repositoryShadow, rootOptions, cause, task, parentResult);
 	}
 

provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/AbstractProvisioningIntegrationTest.java

@@ -291,7 +291,7 @@ protected ShadowAsserter<Void> assertShadowNoFetch(String oid) throws ObjectNotF
 	}
 
 	protected ShadowAsserter<Void> assertShadowFuture(String oid) throws ObjectNotFoundException, CommunicationException, SchemaException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException {
-		Task task = createTask("assertShadowProvisioning");
+		Task task = createTask("assertShadowFuture");
 		OperationResult result = task.getResult();
 		Collection<SelectorOptions<GetOperationOptions>> options =  SelectorOptions.createCollection(GetOperationOptions.createPointInTimeType(PointInTimeType.FUTURE));
 		PrismObject<ShadowType> shadow = provisioningService.getObject(ShadowType.class, oid, options, task, result);
@@ -301,4 +301,18 @@ protected ShadowAsserter<Void> assertShadowFuture(String oid) throws ObjectNotFo
 			.display();
 		return asserter;
 	}
+
+	protected ShadowAsserter<Void> assertShadowFutureNoFetch(String oid) throws ObjectNotFoundException, CommunicationException, SchemaException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException {
+		Task task = createTask("assertShadowFutureNoFetch");
+		OperationResult result = task.getResult();
+		GetOperationOptions rootOptions = GetOperationOptions.createPointInTimeType(PointInTimeType.FUTURE);
+		rootOptions.setNoFetch(true);
+		Collection<SelectorOptions<GetOperationOptions>> options =  SelectorOptions.createCollection(rootOptions);
+		PrismObject<ShadowType> shadow = provisioningService.getObject(ShadowType.class, oid, options, task, result);
+		assertSuccess(result);
+		ShadowAsserter<Void> asserter = ShadowAsserter.forShadow(shadow, "future,noFetch");
+		asserter
+			.display();
+		return asserter;
+	}
 }