Consistency update: more test fixes

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractGroupingManualResourceTest.java

@@ -43,6 +43,7 @@
 import com.evolveum.midpoint.schema.constants.SchemaConstants;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.task.api.Task;
+import com.evolveum.midpoint.test.asserter.ShadowAsserter;
 import com.evolveum.midpoint.util.logging.Trace;
 import com.evolveum.midpoint.util.logging.TraceManager;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ActivationStatusType;
@@ -519,6 +520,10 @@ public void test250RecomputeWillAfter5min() throws Exception {
 		assertAccountWillAfterChangePasswordAndEnableCaseClosed(TEST_NAME, null);
 	}
 
+	/**
+	 * ff 7min. Backing store updated. Grace expired.
+	 * But operation is still in the shadow. Not expired yet.
+	 */
 	@Test
 	public void test272UpdateBackingStoreAndGetAccountWill() throws Exception {
 		final String TEST_NAME = "test272UpdateBackingStoreAndGetAccountWill";
@@ -527,7 +532,7 @@ public void test272UpdateBackingStoreAndGetAccountWill() throws Exception {
 		Task task = createTask(TEST_NAME);
 		OperationResult result = task.getResult();
 
-		//  ff 7min. Refresh. Oldest delta should expire.
+		//  ff 7min. Refresh. Oldest delta over grace. But not expired yet.
 		clockForward("PT7M");
 
 		backingStoreUpdateWill(USER_WILL_FULL_NAME_PIRATE, INTEREST_ONE, ActivationStatusType.ENABLED, USER_WILL_PASSWORD_NEW);
@@ -541,33 +546,89 @@ public void test272UpdateBackingStoreAndGetAccountWill() throws Exception {
 		displayThen(TEST_NAME);
 		assertSuccess(result);
 
-		display("Model shadow", shadowModel);
-		ShadowType shadowTypeProvisioning = shadowModel.asObjectable();
-		assertShadowName(shadowModel, USER_WILL_NAME);
-		assertEquals("Wrong kind (provisioning)", ShadowKindType.ACCOUNT, shadowTypeProvisioning.getKind());
-		assertShadowActivationAdministrativeStatus(shadowModel, ActivationStatusType.ENABLED);
-		assertAttribute(shadowModel, ATTR_USERNAME_QNAME, USER_WILL_NAME);
-		assertAttribute(shadowModel, ATTR_FULLNAME_QNAME, USER_WILL_FULL_NAME_PIRATE);
-		assertAttributeFromBackingStore(shadowModel, ATTR_DESCRIPTION_QNAME, ACCOUNT_WILL_DESCRIPTION_MANUAL);
-		assertShadowPassword(shadowModel);
+		ShadowAsserter<Void> shadowModelAsserter = ShadowAsserter.forShadow(shadowModel, "model")
+			.assertName(USER_WILL_NAME)
+			.assertKind(ShadowKindType.ACCOUNT)
+			.assertAdministrativeStatus(ActivationStatusType.ENABLED)
+			.attributes()
+				.assertValue(ATTR_USERNAME_QNAME, USER_WILL_NAME)
+				.assertValue(ATTR_FULLNAME_QNAME, USER_WILL_FULL_NAME_PIRATE)
+				.end()
+			.pendingOperations()
+				.assertOperations(3)
+				.end();
+		assertAttributeFromBackingStore(shadowModelAsserter, ATTR_DESCRIPTION_QNAME, ACCOUNT_WILL_DESCRIPTION_MANUAL);
+		assertShadowPassword(shadowModelAsserter);
+
+		assertRepoShadow(accountWillOid)
+			.pendingOperations()
+				.assertOperations(3);
+
+		ShadowAsserter<Void> shadowModelFutureAsserter = assertModelShadowFuture(accountWillOid)
+			.assertName(USER_WILL_NAME)
+			.assertKind(ShadowKindType.ACCOUNT)
+			.assertAdministrativeStatus(ActivationStatusType.ENABLED)
+			.attributes()
+				.assertValue(ATTR_USERNAME_QNAME, USER_WILL_NAME)
+				.assertValue(ATTR_FULLNAME_QNAME, USER_WILL_FULL_NAME_PIRATE)
+				.end();
+		assertAttributeFromBackingStore(shadowModelFutureAsserter, ATTR_DESCRIPTION_QNAME, ACCOUNT_WILL_DESCRIPTION_MANUAL);
+		// TODO
+//		assertShadowPassword(shadowProvisioningFuture);
 
-		assertPendingOperationDeltas(shadowModel, 2);
+		assertCase(willLastCaseOid, SchemaConstants.CASE_STATE_CLOSED);
+	}
+
+	/**
+	 * ff 15min. One pending operation exipired.
+	 */
+	@Test
+	public void test273GetAccountWill() throws Exception {
+		final String TEST_NAME = "test273GetAccountWill";
+		displayTestTitle(TEST_NAME);
+		// GIVEN
+		Task task = createTask(TEST_NAME);
+		OperationResult result = task.getResult();
+
+		//  ff 15min. Oldest delta should expire.
+		clockForward("PT15M");
 
-		PrismObject<ShadowType> shadowRepo = repositoryService.getObject(ShadowType.class, accountWillOid, null, result);
-		display("Repo shadow", shadowRepo);
-		assertPendingOperationDeltas(shadowRepo, 2);
+		// WHEN
+		displayWhen(TEST_NAME);
+		PrismObject<ShadowType> shadowModel = modelService.getObject(ShadowType.class,
+				accountWillOid, null, task, result);
 
-		PrismObject<ShadowType> shadowModelFuture = modelService.getObject(ShadowType.class,
-				accountWillOid,
-				SelectorOptions.createCollection(GetOperationOptions.createPointInTimeType(PointInTimeType.FUTURE)),
-				task, result);
-		display("Model shadow (future)", shadowModelFuture);
-		assertShadowName(shadowModelFuture, USER_WILL_NAME);
-		assertEquals("Wrong kind (provisioning)", ShadowKindType.ACCOUNT, shadowModelFuture.asObjectable().getKind());
-		assertShadowActivationAdministrativeStatus(shadowModelFuture, ActivationStatusType.ENABLED);
-		assertAttribute(shadowModelFuture, ATTR_USERNAME_QNAME, USER_WILL_NAME);
-		assertAttribute(shadowModelFuture, ATTR_FULLNAME_QNAME, USER_WILL_FULL_NAME_PIRATE);
-		assertAttributeFromBackingStore(shadowModelFuture, ATTR_DESCRIPTION_QNAME, ACCOUNT_WILL_DESCRIPTION_MANUAL);
+		// THEN
+		displayThen(TEST_NAME);
+		assertSuccess(result);
+
+		ShadowAsserter<Void> shadowModelAsserter = ShadowAsserter.forShadow(shadowModel, "model")
+			.assertName(USER_WILL_NAME)
+			.assertKind(ShadowKindType.ACCOUNT)
+			.assertAdministrativeStatus(ActivationStatusType.ENABLED)
+			.attributes()
+				.assertValue(ATTR_USERNAME_QNAME, USER_WILL_NAME)
+				.assertValue(ATTR_FULLNAME_QNAME, USER_WILL_FULL_NAME_PIRATE)
+				.end()
+			.pendingOperations()
+				.assertOperations(2)
+				.end();
+		assertAttributeFromBackingStore(shadowModelAsserter, ATTR_DESCRIPTION_QNAME, ACCOUNT_WILL_DESCRIPTION_MANUAL);
+		assertShadowPassword(shadowModelAsserter);
+
+		assertRepoShadow(accountWillOid)
+			.pendingOperations()
+				.assertOperations(2);
+
+		ShadowAsserter<Void> shadowModelFutureAsserter = assertModelShadowFuture(accountWillOid)
+			.assertName(USER_WILL_NAME)
+			.assertKind(ShadowKindType.ACCOUNT)
+			.assertAdministrativeStatus(ActivationStatusType.ENABLED)
+			.attributes()
+				.assertValue(ATTR_USERNAME_QNAME, USER_WILL_NAME)
+				.assertValue(ATTR_FULLNAME_QNAME, USER_WILL_FULL_NAME_PIRATE)
+				.end();
+		assertAttributeFromBackingStore(shadowModelFutureAsserter, ATTR_DESCRIPTION_QNAME, ACCOUNT_WILL_DESCRIPTION_MANUAL);
 		// TODO
 //		assertShadowPassword(shadowProvisioningFuture);
 

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractManualResourceTest.java

@@ -2143,7 +2143,7 @@ protected void assertAccountWillAfterAssign(final String TEST_NAME, String expec
 				.singleOperation()
 					.assertId()
 					.assertRequestTimestamp(accountWillReqestTimestampStart, accountWillReqestTimestampEnd)
-					.assertExecutionStatus(PendingOperationExecutionStatusType.EXECUTING)
+					.assertExecutionStatus(getExpectedExecutionStatus(propagationExecutionStage))
 					.delta()
 						.display()
 						.end()
@@ -2167,7 +2167,7 @@ protected void assertAccountWillAfterAssign(final String TEST_NAME, String expec
 			.pendingOperations()
 				.singleOperation()
 					.assertRequestTimestamp(accountWillReqestTimestampStart, accountWillReqestTimestampEnd)
-					.assertExecutionStatus(PendingOperationExecutionStatusType.EXECUTING)
+					.assertExecutionStatus(getExpectedExecutionStatus(propagationExecutionStage))
 					.end()
 				.end();
 		assertAttributeFromCache(shadowModelAsserter, ATTR_FULLNAME_QNAME, expectedFullName);

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/TestSemiManualGrouping.java

@@ -43,6 +43,7 @@
 import com.evolveum.midpoint.schema.constants.SchemaConstants;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.task.api.Task;
+import com.evolveum.midpoint.test.asserter.ShadowAsserter;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultStatusType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.PendingOperationExecutionStatusType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.PendingOperationType;
@@ -134,7 +135,7 @@ protected void assertShadowPassword(PrismObject<ShadowType> shadow) {
 	 * Create phantom account in the backing store. MidPoint does not know anything about it.
 	 * At the same time, there is phantom user that has the account assigned. But it is not yet
 	 * provisioned. MidPoint won't figure out that there is already an account, as the propagation
-	 * is not execute immediatelly. The conflict will be discovered only later, when propagation
+	 * is not executed immediately. The conflict will be discovered only later, when propagation
 	 * task is run.
 	 * MID-4614
 	 */
@@ -160,27 +161,28 @@ public void test400PhantomAccount() throws Exception {
 		// No case OID yet. The case would be created after propagation is run.
 		assertNull("Unexpected case 1 OID", caseOid1);
 
-		PrismObject<UserType> userMid1 = getUser(USER_PHANTOM_OID);
-		display("User mid1", userMid1);
-		String shadowOid = getSingleLinkOid(userMid1);
-		PrismObject<ShadowType> shadowMid1NoFetch = getShadowModelNoFetch(shadowOid);
-		display("Shadow mid1 (model, noFetch)", shadowMid1NoFetch);
-
-		assertAttribute(shadowMid1NoFetch, ATTR_USERNAME_QNAME, USER_PHANTOM_USERNAME);
-
-		PendingOperationType pendingOperation = assertSinglePendingOperation(shadowMid1NoFetch, 
-				PendingOperationExecutionStatusType.EXECUTION_PENDING, null);
+		String shadowOid = assertUser(USER_PHANTOM_OID, "mid1")
+			.singleLink()
+				.getOid();
+
+		assertModelShadowNoFetch(shadowOid)
+			.attributes()
+				.assertValue(ATTR_USERNAME_QNAME, USER_PHANTOM_USERNAME)
+				.end()
+			.pendingOperations()
+				.singleOperation()
+					.assertExecutionStatus(PendingOperationExecutionStatusType.EXECUTION_PENDING);
 
 		clockForward("PT3M");
 
 		// WHEN (mid2)
 		displayWhen(TEST_NAME, "mid2");
 		// Existing account is detected now. Hence partial error.		
-		runPropagation();
+		runPropagation(OperationResultStatusType.PARTIAL_ERROR);
 
 		// Synchronization service kicks in, reconciliation detects wrong full name.
 		// It tries to fix it. But as we are in propagation mode, the data are not
-		// fixed immediatelly. Instead there is a pending delta to fix the problem.
+		// fixed immediately. Instead there is a pending delta to fix the problem.
 
 		// THEN (mid2)
 		displayThen(TEST_NAME, "mid2");
@@ -189,24 +191,47 @@ public void test400PhantomAccount() throws Exception {
 		// No case OID yet. The case will be created after propagation is run.
 		assertNull("Unexpected case 2 OID", caseOid2);
 
-		PrismObject<UserType> userMid2 = getUser(USER_PHANTOM_OID);
-		display("User mid2", userMid2);
-		String shadowOidMid2 = getSingleLinkOid(userMid2);
-		PrismObject<ShadowType> shadowMid2NoFetch = getShadowModelNoFetch(shadowOidMid2);
-		display("Shadow mid2 (model, noFetch)", shadowMid2NoFetch);
-
-		assertAttribute(shadowMid2NoFetch, ATTR_USERNAME_QNAME, USER_PHANTOM_USERNAME);
-
-		assertPendingOperationDeltas(shadowMid2NoFetch, 2);
-		PendingOperationType fizzledAddOperation = findPendingOperation(shadowMid2NoFetch, OperationResultStatusType.HANDLED_ERROR, ChangeTypeType.ADD);
-		assertPendingOperation(shadowMid2NoFetch, fizzledAddOperation,
-						PendingOperationExecutionStatusType.COMPLETED, OperationResultStatusType.HANDLED_ERROR);
-		assertNotNull("Null completion timestamp", fizzledAddOperation.getCompletionTimestamp());
-		PendingOperationType reconOperation = findPendingOperation(shadowMid2NoFetch, null, ChangeTypeType.MODIFY);
-		assertPendingOperation(shadowMid2NoFetch, reconOperation,
-				PendingOperationExecutionStatusType.EXECUTION_PENDING, null);
-		assertHasModification(reconOperation.getDelta(), new ItemPath(ShadowType.F_ATTRIBUTES, new QName(MidPointConstants.NS_RI, "fullname")));
-
+		assertUser(USER_PHANTOM_OID, "mid2")
+				.displayWithProjections()
+				.links()
+					.assertLinks(2)
+					.by()
+						.dead(true)
+					.find()
+						.resolveTarget()
+							.assertTombstone()
+							.pendingOperations()
+								.singleOperation()
+									.assertExecutionStatus(PendingOperationExecutionStatusType.COMPLETED)
+									// Fatal error. Add operation failed. It could not proceed as there was existing account already.
+									.assertResultStatus(OperationResultStatusType.FATAL_ERROR)
+									.assertHasCompletionTimestamp()
+									.delta()
+										.assertAdd()
+										.end()
+									.end()
+								.end()
+							.end()
+						.end()
+					.by()
+						.dead(false)
+					.find()
+						.resolveTarget()
+							.assertLife()
+							.pendingOperations()
+								.singleOperation()
+									.assertExecutionStatus(PendingOperationExecutionStatusType.EXECUTION_PENDING)
+									.delta()
+										.assertModify()
+										.assertHasModification(new ItemPath(ShadowType.F_ATTRIBUTES, new QName(MidPointConstants.NS_RI, "fullname")))
+										.end()
+									.end()
+								.end()
+							.end()
+						.end()
+					.end()
+				.end();
+
 		clockForward("PT20M");
 
 		// WHEN (final)
@@ -216,24 +241,57 @@ public void test400PhantomAccount() throws Exception {
 		// THEN
 		displayThen(TEST_NAME, "final");
 
-		PrismObject<UserType> userAfter = getUser(USER_PHANTOM_OID);
-		display("User after", userAfter);
-		shadowOid = getSingleLinkOid(userAfter);
-		PrismObject<ShadowType> shadowModel = getShadowModel(shadowOid);
-		display("Shadow after (model)", shadowModel);
-
-		assertAttribute(shadowModel, ATTR_USERNAME_QNAME, USER_PHANTOM_USERNAME);
-		assertAttribute(shadowModel, ATTR_FULLNAME_QNAME, USER_PHANTOM_FULL_NAME_WRONG);
-		assertAttributeFromBackingStore(shadowModel, ATTR_DESCRIPTION_QNAME, ACCOUNT_PHANTOM_DESCRIPTION_MANUAL);
-		assertShadowPassword(shadowModel);
-
-		assertPendingOperationDeltas(shadowModel, 1);
-//		fizzledAddOperation = findPendingOperation(shadowModel, OperationResultStatusType.HANDLED_ERROR, ChangeTypeType.ADD);
-//		assertPendingOperation(shadowModel, fizzledAddOperation,
-//						PendingOperationExecutionStatusType.COMPLETED, OperationResultStatusType.HANDLED_ERROR);
-		reconOperation = findPendingOperation(shadowModel, null, ChangeTypeType.MODIFY);
-		assertPendingOperation(shadowModel, reconOperation,
-				PendingOperationExecutionStatusType.EXECUTING, OperationResultStatusType.IN_PROGRESS);
+		String liveShadowOid = assertUser(USER_PHANTOM_OID, "final")
+			.displayWithProjections()
+			.links()
+				.assertLinks(2)
+				.by()
+					.dead(true)
+				.find()
+					.resolveTarget()
+						.assertTombstone()
+						.pendingOperations()
+							.singleOperation()
+								.assertExecutionStatus(PendingOperationExecutionStatusType.COMPLETED)
+								.assertResultStatus(OperationResultStatusType.FATAL_ERROR)
+								.assertHasCompletionTimestamp()
+								.delta()
+									.assertAdd()
+									.end()
+								.end()
+							.end()
+						.end()
+					.end()
+				.by()
+					.dead(false)
+				.find()
+					.getOid();
+
+		ShadowAsserter<Void> shadowModelAfterAsserter = assertModelShadow(liveShadowOid)
+			.assertLife()
+			.attributes()
+				.assertValue(ATTR_USERNAME_QNAME, USER_PHANTOM_USERNAME)
+				.assertValue(ATTR_FULLNAME_QNAME, USER_PHANTOM_FULL_NAME_WRONG)
+				.end()
+			.pendingOperations()
+				.singleOperation()
+					.assertExecutionStatus(PendingOperationExecutionStatusType.EXECUTING)
+					.assertResultStatus(OperationResultStatusType.IN_PROGRESS)
+					.delta()
+						.assertModify()
+						.end()
+					.end()
+				.end();
+		assertAttributeFromBackingStore(shadowModelAfterAsserter, ATTR_DESCRIPTION_QNAME, ACCOUNT_PHANTOM_DESCRIPTION_MANUAL);
+		assertShadowPassword(shadowModelAfterAsserter);
+
+		ShadowAsserter<Void> shadowModelAfterAsserterFuture = assertModelShadowFuture(liveShadowOid)
+			.attributes()
+				.assertValue(ATTR_USERNAME_QNAME, USER_PHANTOM_USERNAME)
+				.assertValue(ATTR_FULLNAME_QNAME, USER_PHANTOM_FULL_NAME)
+				.end();
+		assertAttributeFromBackingStore(shadowModelAfterAsserterFuture, ATTR_DESCRIPTION_QNAME, ACCOUNT_PHANTOM_DESCRIPTION_MANUAL);
+		assertShadowPassword(shadowModelAfterAsserterFuture);
 
 		// TODO: assert the case
 

model/model-intest/src/test/resources/logback-test.xml

@@ -80,11 +80,11 @@
  	<logger name="com.evolveum.midpoint.model.impl.util" level="DEBUG" />
 	<logger name="com.evolveum.midpoint.model.impl.sync" level="DEBUG" />
  	<logger name="com.evolveum.midpoint.model.impl.sync.CorrelationConfirmationEvaluator" level="DEBUG" />
-    <logger name="com.evolveum.midpoint.provisioning" level="DEBUG" />
+    <logger name="com.evolveum.midpoint.provisioning" level="TRACE" />
     <logger name="com.evolveum.midpoint.provisioning.impl.ResourceManager" level="DEBUG" />
     <logger name="com.evolveum.midpoint.provisioning.impl.ShadowCache" level="TRACE" />
     <logger name="com.evolveum.midpoint.provisioning.impl.ShadowManager" level="TRACE" />
-    <logger name="com.evolveum.midpoint.provisioning.impl.task" level="DEBUG" />
+    <logger name="com.evolveum.midpoint.provisioning.impl.task" level="TRACE" />
     <logger name="com.evolveum.midpoint.provisioning.impl.errorhandling" level="TRACE" />
 	<logger name="com.evolveum.midpoint.expression" level="DEBUG" />
 	<logger name="com.evolveum.midpoint.model.common.expression" level="DEBUG" />
@@ -113,10 +113,10 @@
     <logger name="com.evolveum.icf.dummy" level="INFO" />
     <logger name="com.evolveum.midpoint.model.impl.expr" level="DEBUG" />
     <logger name="com.evolveum.midpoint.model.impl.util.DeleteTaskHandler" level="DEBUG" />
-	<logger name="com.evolveum.midpoint.repo" level="DEBUG" />
+	<logger name="com.evolveum.midpoint.repo" level="INFO" />
 	<logger name="com.evolveum.midpoint.repo.operation" level="DEBUG" />
-	<logger name="com.evolveum.midpoint.repo.sql.util.MidpointPersisterUtil" level="DEBUG" />
-	<logger name="com.evolveum.midpoint.repo.cache" level="DEBUG" />
+	<logger name="com.evolveum.midpoint.repo.sql.util.MidpointPersisterUtil" level="INFO" />
+	<logger name="com.evolveum.midpoint.repo.cache" level="INFO" />
 	<logger name="com.evolveum.midpoint.repo.common.task" level="DEBUG" />
 	<logger name="com.evolveum.midpoint.model.common.SystemObjectCache" level="DEBUG" />
 	<logger name="com.evolveum.midpoint.model.impl.controller.AuditController" level="DEBUG" />

model/model-intest/src/test/resources/manual/resource-manual-grouping.xml

@@ -130,6 +130,8 @@
 		<shadowConstraintsCheck>light</shadowConstraintsCheck>
 		<pendingOperationGracePeriod>PT15M</pendingOperationGracePeriod>
 		<operationGroupingInterval>PT2M</operationGroupingInterval>
+		<pendingOperationRetentionPeriod>PT30M</pendingOperationRetentionPeriod>
+		<deadShadowRetentionPeriod>PT2H</deadShadowRetentionPeriod>
 		<reshreshOnRead>true</reshreshOnRead>
 	</consistency>