Consistency update: reshreshOnRead configuraiton (MID-3891)

infra/schema/src/main/java/com/evolveum/midpoint/schema/util/ResourceTypeUtil.java

@@ -61,6 +61,7 @@
 import com.evolveum.midpoint.xml.ns._public.resource.capabilities_3.TestConnectionCapabilityType;
 import com.evolveum.midpoint.xml.ns._public.resource.capabilities_3.UpdateCapabilityType;
 import com.evolveum.prism.xml.ns._public.types_3.SchemaDefinitionType;
+import com.sun.tools.xjc.reader.RawTypeSet.Ref;
 
 /**
  * Methods that would belong to the ResourceType class but cannot go there
@@ -706,4 +707,16 @@ public static RecordPendingOperationsType getRecordPendingOperations(ResourceTyp
 		}
 		return recordPendingOperations;
 	}
+
+	public static boolean isRefreshOnRead(ResourceType resource) {
+		ResourceConsistencyType consistency = resource.getConsistency();
+		if (consistency == null) {
+			return false;
+		}
+		Boolean reshreshOnRead = consistency.isReshreshOnRead();
+		if (reshreshOnRead == null) {
+			return false;
+		}
+		return reshreshOnRead;
+	}
 }

infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd

@@ -6122,6 +6122,24 @@
             		</xsd:appinfo>
             	</xsd:annotation>
             </xsd:element>
+            <xsd:element name="reshreshOnRead" type="xsd:boolean" minOccurs="0" default="false">
+            	<xsd:annotation>
+            		<xsd:documentation>
+            			If set to true then midPoint will always refresh shadow when it is retrieved.
+            			Refreshing a shadow means that the status of asynchronous (e.g. manual) opearions
+            			is checked, unfinished operations may be retried and so on. In this case shadow
+            			will always be as fresh as it can be. But read may be slower and there may be
+            			strange errors (e.g. reading a shadow may cause "already exists" error because
+            			pending ADD operation in the shadow was executed during that read).
+            			If set to false (which is the default) then refresh will not be executed during
+            			read operations - unless the refresh is explicitly requested by midPoint code
+            			(e.g. during reconciliation).
+            		</xsd:documentation>
+            		<xsd:appinfo>
+            			<a:since>3.9</a:since>
+            		</xsd:appinfo>
+            	</xsd:annotation>
+            </xsd:element>
         </xsd:sequence>
         <xsd:attribute name="id" type="xsd:long" use="optional"/>
     </xsd:complexType>

provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java

@@ -203,7 +203,8 @@ public PrismObject<ShadowType> getShadow(String oid, PrismObject<ShadowType> rep
 			throw e;
 		}
 
-		if (GetOperationOptions.isForceRefresh(rootOptions)) {
+		if (shouldRefreshOnRead(resource, rootOptions)) {
+			LOGGER.trace("Refreshing shadow {} before reading", repositoryShadow);
 			repositoryShadow = refreshShadow(repositoryShadow, task, parentResult);
 		}
 		if (repositoryShadow == null) {
@@ -356,6 +357,10 @@ public PrismObject<ShadowType> getShadow(String oid, PrismObject<ShadowType> rep
 		}
 	}
 
+	private boolean shouldRefreshOnRead(ResourceType resource, GetOperationOptions rootOptions) {
+		return GetOperationOptions.isForceRefresh(rootOptions) || ResourceTypeUtil.isRefreshOnRead(resource);
+	}
+
 	private PrismObject<ShadowType> processNoFetchGet(ProvisioningContext ctx,
 			PrismObject<ShadowType> repositoryShadow, Collection<SelectorOptions<GetOperationOptions>> options, 
 			XMLGregorianCalendar now, Task task, OperationResult parentResult) 

provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/manual/AbstractManualResourceTest.java

@@ -1415,6 +1415,7 @@ public void test230ModifyAccountWillChangePasswordAndEnable() throws Exception {
 	 * Disable case is closed. The account should be disabled. But there is still the other
 	 * delta pending.
 	 * Do NOT explicitly refresh the shadow in this case. Just reading it should cause the refresh.
+	 * Even though forceRefresh is NOT used here. The resource is set to automatic refresh.
 	 */
 	@Test
 	public void test240CloseDisableCaseAndReadAccountWill() throws Exception {
@@ -1425,12 +1426,9 @@ public void test240CloseDisableCaseAndReadAccountWill() throws Exception {
 		OperationResult result = task.getResult();
 		syncServiceMock.reset();
 
-		accountWillCompletionTimestampStart = clock.currentTimeXMLGregorianCalendar();
-
 		closeCase(willLastCaseOid);
-
-		PrismObject<ShadowType> shadowBefore = provisioningService.getObject(ShadowType.class, ACCOUNT_WILL_OID, null, task, result);
-		display("Shadow before", shadowBefore);
+
+		accountWillCompletionTimestampStart = clock.currentTimeXMLGregorianCalendar();
 
 		// WHEN
 		displayWhen(TEST_NAME);
@@ -1440,72 +1438,93 @@ public void test240CloseDisableCaseAndReadAccountWill() throws Exception {
 		// THEN
 		displayThen(TEST_NAME);
 		assertSuccess(result);
-
+		
 		accountWillCompletionTimestampEnd = clock.currentTimeXMLGregorianCalendar();
 
-		PrismObject<ShadowType> shadowRepo = getShadowRepo(ACCOUNT_WILL_OID);
-		display("Repo shadow", shadowRepo);
-
-		assertPendingOperationDeltas(shadowRepo, 3);
-
-		PendingOperationType pendingOperation = findPendingOperation(shadowRepo,
-				OperationResultStatusType.SUCCESS, SchemaConstants.PATH_ACTIVATION_ADMINISTRATIVE_STATUS);
-		assertPendingOperation(shadowRepo, pendingOperation,
-				accountWillReqestTimestampStart, accountWillReqestTimestampEnd,
-				OperationResultStatusType.SUCCESS,
-				accountWillCompletionTimestampStart, accountWillCompletionTimestampEnd);
+		display("Provisioning shadow", shadowProvisioning);
 
-		pendingOperation = findPendingOperation(shadowRepo,
-				OperationResultStatusType.IN_PROGRESS, SchemaConstants.PATH_PASSWORD_VALUE);
-		assertPendingOperation(shadowRepo, pendingOperation, accountWillSecondReqestTimestampStart, accountWillSecondReqestTimestampEnd);
+		PrismObject<ShadowType> shadowRepo = assertRepoShadow(ACCOUNT_WILL_OID)
+			.pendingOperations()
+				.assertOperations(3)
+				.by()
+					.executionStatus(PendingOperationExecutionStatusType.COMPLETED)
+					.item(SchemaConstants.PATH_ACTIVATION_ADMINISTRATIVE_STATUS)
+				.find()
+					.assertRequestTimestamp(accountWillReqestTimestampStart, accountWillReqestTimestampEnd)
+					.assertResultStatus(OperationResultStatusType.SUCCESS)
+					.assertCompletionTimestamp(accountWillCompletionTimestampStart, accountWillCompletionTimestampEnd)
+					.end()
+				.by()
+					.executionStatus(PendingOperationExecutionStatusType.EXECUTING)
+					.item(SchemaConstants.PATH_PASSWORD_VALUE)
+				.find()
+					.assertRequestTimestamp(accountWillSecondReqestTimestampStart, accountWillSecondReqestTimestampEnd)
+					.assertResultStatus(OperationResultStatusType.IN_PROGRESS)
+					.end()
+				.end()
+			.attributes()
+				.assertValue(ATTR_USERNAME_QNAME, ACCOUNT_WILL_USERNAME)
+				.end()
+			.getObject();
 
 		assertShadowActivationAdministrativeStatusFromCache(shadowRepo, ActivationStatusType.DISABLED);
-		assertAttribute(shadowRepo, ATTR_USERNAME_QNAME, ACCOUNT_WILL_USERNAME);
 		assertAttributeFromCache(shadowRepo, ATTR_FULLNAME_QNAME, ACCOUNT_WILL_FULLNAME_PIRATE);
 
 		syncServiceMock.assertNoNotifyChange();
 		syncServiceMock.assertNotifySuccessOnly();
 
-		display("Provisioning shadow", shadowProvisioning);
-		ShadowType shadowTypeProvisioning = shadowProvisioning.asObjectable();
-		assertShadowName(shadowProvisioning, ACCOUNT_WILL_USERNAME);
-		assertEquals("Wrong kind (provisioning)", ShadowKindType.ACCOUNT, shadowTypeProvisioning.getKind());
+		ShadowAsserter.forShadow(shadowProvisioning, "provisioning")
+			.assertName(ACCOUNT_WILL_USERNAME)
+			.assertKind(ShadowKindType.ACCOUNT)
+			.attributes()
+				.assertValue(ATTR_USERNAME_QNAME, ACCOUNT_WILL_USERNAME)
+				.assertValue(ATTR_FULLNAME_QNAME, ACCOUNT_WILL_FULLNAME_PIRATE)
+			.end()
+			.pendingOperations()
+				.assertOperations(3)
+				.by()
+					.executionStatus(PendingOperationExecutionStatusType.COMPLETED)
+					.item(SchemaConstants.PATH_ACTIVATION_ADMINISTRATIVE_STATUS)
+				.find()
+					.assertRequestTimestamp(accountWillReqestTimestampStart, accountWillReqestTimestampEnd)
+					.assertResultStatus(OperationResultStatusType.SUCCESS)
+					.assertCompletionTimestamp(accountWillCompletionTimestampStart, accountWillCompletionTimestampEnd)
+					.end()
+				.by()
+					.executionStatus(PendingOperationExecutionStatusType.EXECUTING)
+					.item(SchemaConstants.PATH_PASSWORD_VALUE)
+				.find()
+					.assertRequestTimestamp(accountWillSecondReqestTimestampStart, accountWillSecondReqestTimestampEnd)
+					.assertResultStatus(OperationResultStatusType.IN_PROGRESS)
+					.end()
+				.end();
+
 		if (supportsBackingStore()) {
 			assertShadowActivationAdministrativeStatus(shadowProvisioning, ActivationStatusType.ENABLED);
 		} else {
 			assertShadowActivationAdministrativeStatus(shadowProvisioning, ActivationStatusType.DISABLED);
 		}
-
-		assertAttribute(shadowProvisioning, ATTR_USERNAME_QNAME, ACCOUNT_WILL_USERNAME);
-		assertAttribute(shadowProvisioning, ATTR_FULLNAME_QNAME, ACCOUNT_WILL_FULLNAME_PIRATE);
 		assertAttributeFromBackingStore(shadowProvisioning, ATTR_DESCRIPTION_QNAME, ACCOUNT_WILL_DESCRIPTION_MANUAL);
 		assertShadowPassword(shadowProvisioning);
 
-		assertPendingOperationDeltas(shadowProvisioning, 3);
-		pendingOperation = findPendingOperation(shadowProvisioning,
-				OperationResultStatusType.SUCCESS, SchemaConstants.PATH_ACTIVATION_ADMINISTRATIVE_STATUS);
-		assertPendingOperation(shadowProvisioning, pendingOperation,
-				accountWillReqestTimestampStart, accountWillReqestTimestampEnd,
-				OperationResultStatusType.SUCCESS,
-				accountWillCompletionTimestampStart, accountWillCompletionTimestampEnd);
-
-		PrismObject<ShadowType> shadowProvisioningFuture = provisioningService.getObject(ShadowType.class,
-				ACCOUNT_WILL_OID,
-				SelectorOptions.createCollection(GetOperationOptions.createPointInTimeType(PointInTimeType.FUTURE)),
-				task, result);
-		display("Provisioning shadow (future)", shadowProvisioningFuture);
-		assertShadowName(shadowProvisioningFuture, ACCOUNT_WILL_USERNAME);
-		assertEquals("Wrong kind (provisioning)", ShadowKindType.ACCOUNT, shadowProvisioningFuture.asObjectable().getKind());
-		assertShadowActivationAdministrativeStatus(shadowProvisioningFuture, ActivationStatusType.ENABLED);
-		assertAttribute(shadowProvisioningFuture, ATTR_USERNAME_QNAME, ACCOUNT_WILL_USERNAME);
-		assertAttribute(shadowProvisioningFuture, ATTR_FULLNAME_QNAME, ACCOUNT_WILL_FULLNAME_PIRATE);
+		PrismObject<ShadowType> shadowProvisioningFuture = assertShadowFuture(ACCOUNT_WILL_OID)
+			.assertName(ACCOUNT_WILL_USERNAME)
+			.assertKind(ShadowKindType.ACCOUNT)
+			.assertAdministrativeStatus(ActivationStatusType.ENABLED)
+			.attributes()
+				.assertValue(ATTR_USERNAME_QNAME, ACCOUNT_WILL_USERNAME)
+				.assertValue(ATTR_FULLNAME_QNAME, ACCOUNT_WILL_FULLNAME_PIRATE)
+				.end()
+			.getObject();
+
 		assertAttributeFromBackingStore(shadowProvisioningFuture, ATTR_DESCRIPTION_QNAME, ACCOUNT_WILL_DESCRIPTION_MANUAL);
 		// TODO
 //		assertShadowPassword(shadowProvisioningFuture);
 
 		assertCase(willLastCaseOid, SchemaConstants.CASE_STATE_CLOSED);
 		assertCase(willSecondLastCaseOid, SchemaConstants.CASE_STATE_OPEN);
 	}
+
 
 	/**
 	 * lets ff 5min just for fun. Refresh, make sure everything should be the same (grace not expired yet)

provisioning/provisioning-impl/src/test/resources/manual/resource-manual.xml

@@ -81,6 +81,7 @@
 	<consistency>
 		<pendingOperationGracePeriod>PT15M</pendingOperationGracePeriod>
 		<pendingOperationRetentionPeriod>PT20M</pendingOperationRetentionPeriod>
+		<reshreshOnRead>true</reshreshOnRead>
 	</consistency>
 
 	<business>

provisioning/provisioning-impl/src/test/resources/manual/resource-semi-manual.xml

@@ -80,6 +80,7 @@
 	<consistency>
 		<pendingOperationGracePeriod>PT15M</pendingOperationGracePeriod>
 		<pendingOperationRetentionPeriod>PT20M</pendingOperationRetentionPeriod>
+		<reshreshOnRead>true</reshreshOnRead>
 	</consistency>
 
 </resource>