Stop updating sync token in dry run (MID-5644)

When doing LiveSync in dry run mode we no longer update the token (by default); with the exception of token initialization. The original behavior of updating the token can be enabled by setting updateLiveSyncTokenInDryRun extension property to true.
Evolveum · Oct 2, 2019 · 4bf445a · 4bf445a
1 parent af385d4
commit 4bf445a
Show file tree

Hide file tree

Showing 11 changed files with 226 additions and 66 deletions.
diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java
@@ -275,6 +275,7 @@ public abstract class SchemaConstants {
 	public static final ItemName MODEL_EXTENSION_DRY_RUN = new ItemName(NS_MODEL_EXTENSION, "dryRun");
 	public static final ItemName MODEL_EXTENSION_SIMULATE_BEFORE_EXECUTE = new ItemName(NS_MODEL_EXTENSION, "simulateBeforeExecute");
 	public static final ItemName MODEL_EXTENSION_RETRY_LIVE_SYNC_ERRORS = new ItemName(NS_MODEL_EXTENSION, "retryLiveSyncErrors");
+	public static final ItemName MODEL_EXTENSION_UPDATE_LIVE_SYNC_TOKEN_IN_DRY_RUN = new ItemName(NS_MODEL_EXTENSION, "updateLiveSyncTokenInDryRun");
 	public static final ItemName MODEL_EXTENSION_LIVE_SYNC_BATCH_SIZE = new ItemName(NS_MODEL_EXTENSION, "liveSyncBatchSize");
 	public static final ItemName MODEL_EXTENSION_FINISH_OPERATIONS_ONLY = new ItemName(NS_MODEL_EXTENSION, "finishOperationsOnly");
 	public static final ItemName MODEL_EXTENSION_KIND = new ItemName(NS_MODEL_EXTENSION, "kind");

diff --git a/infra/schema/src/main/resources/xml/ns/public/model/extension-3.xsd b/infra/schema/src/main/resources/xml/ns/public/model/extension-3.xsd
@@ -90,6 +90,22 @@
         </xsd:annotation>
     </xsd:element>
 
+    <xsd:element name="updateLiveSyncTokenInDryRun" type="xsd:boolean">
+        <xsd:annotation>
+            <xsd:documentation>
+                Indicates if the LiveSync token should be updated when running in dry run mode. If false (the default) then
+                LiveSync will not update token to a new value so it will process objects fetched on the next run (either dry
+                or normal). If true, it will update the token, and therefore marks objects as processed.
+            </xsd:documentation>
+            <xsd:appinfo>
+                <a:displayName>Retry LiveSync Errors</a:displayName>
+                <a:displayOrder>801</a:displayOrder>
+           	 	<a:minOccurs>0</a:minOccurs>
+            	<a:maxOccurs>1</a:maxOccurs>
+            </xsd:appinfo>
+        </xsd:annotation>
+    </xsd:element>
+
     <!-- we might consider making this part of resource configuration -->
     <xsd:element name="liveSyncBatchSize" type="xsd:int">
         <xsd:annotation>

diff --git a/...in/java/com/evolveum/midpoint/model/impl/integrity/ShadowIntegrityCheckResultHandler.java b/...in/java/com/evolveum/midpoint/model/impl/integrity/ShadowIntegrityCheckResultHandler.java
@@ -13,7 +13,6 @@
 import com.evolveum.midpoint.model.common.SystemObjectCache;
 import com.evolveum.midpoint.model.impl.sync.SynchronizationContext;
 import com.evolveum.midpoint.model.impl.sync.SynchronizationService;
-import com.evolveum.midpoint.model.impl.util.ModelImplUtils;
 import com.evolveum.midpoint.prism.*;
 import com.evolveum.midpoint.prism.delta.*;
 import com.evolveum.midpoint.prism.equivalence.EquivalenceStrategy;
@@ -33,6 +32,7 @@
 import com.evolveum.midpoint.task.api.RunningTask;
 import com.evolveum.midpoint.task.api.Task;
 import com.evolveum.midpoint.task.api.TaskManager;
+import com.evolveum.midpoint.task.api.TaskUtil;
 import com.evolveum.midpoint.util.DebugUtil;
 import com.evolveum.midpoint.util.exception.*;
 import com.evolveum.midpoint.util.logging.LoggingUtils;
@@ -195,7 +195,7 @@ public ShadowIntegrityCheckResultHandler(RunningTask coordinatorTask, String tas
         }
 
         try {
-            dryRun = ModelImplUtils.isDryRun(coordinatorTask);
+            dryRun = TaskUtil.isDryRun(coordinatorTask);
         } catch (SchemaException e) {
             throw new SystemException("Couldn't get dryRun flag from task " + coordinatorTask);
         }

diff --git a/...l-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/ReconciliationTaskHandler.java b/...l-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/ReconciliationTaskHandler.java
@@ -15,6 +15,7 @@
 import com.evolveum.midpoint.model.impl.util.AuditHelper;
 import com.evolveum.midpoint.prism.query.AndFilter;
 import com.evolveum.midpoint.schema.cache.CacheConfigurationManager;
+import com.evolveum.midpoint.task.api.*;
 import com.evolveum.prism.xml.ns._public.query_3.QueryType;
 import org.apache.commons.lang.BooleanUtils;
 import org.jetbrains.annotations.NotNull;
@@ -37,8 +38,6 @@
 import com.evolveum.midpoint.prism.PrismContext;
 import com.evolveum.midpoint.prism.PrismObject;
 import com.evolveum.midpoint.prism.PrismProperty;
-import com.evolveum.midpoint.prism.delta.ChangeType;
-import com.evolveum.midpoint.prism.delta.ItemDelta;
 import com.evolveum.midpoint.prism.delta.ObjectDelta;
 import com.evolveum.midpoint.prism.query.ObjectQuery;
 import com.evolveum.midpoint.prism.xml.XmlTypeConverter;
@@ -60,15 +59,7 @@
 import com.evolveum.midpoint.schema.result.OperationResultStatus;
 import com.evolveum.midpoint.schema.util.ObjectTypeUtil;
 import com.evolveum.midpoint.schema.util.ShadowUtil;
-import com.evolveum.midpoint.task.api.RunningTask;
-import com.evolveum.midpoint.task.api.StatisticsCollectionStrategy;
-import com.evolveum.midpoint.task.api.Task;
-import com.evolveum.midpoint.task.api.TaskCategory;
-import com.evolveum.midpoint.task.api.TaskManager;
-import com.evolveum.midpoint.task.api.TaskRunResult;
 import com.evolveum.midpoint.task.api.TaskRunResult.TaskRunResultStatus;
-import com.evolveum.midpoint.task.api.TaskWorkBucketProcessingResult;
-import com.evolveum.midpoint.task.api.WorkBucketAwareTaskHandler;
 import com.evolveum.midpoint.util.Holder;
 import com.evolveum.midpoint.util.QNameUtil;
 import com.evolveum.midpoint.util.exception.CommunicationException;
@@ -708,7 +699,7 @@ private PrismObject<ShadowType> reconcileShadow(PrismObject<ShadowType> shadow,
 		OperationResult opResult = new OperationResult(OperationConstants.RECONCILIATION+".shadowReconciliation.object");
 		try {
 			Collection<SelectorOptions<GetOperationOptions>> options = null;
-			if (ModelImplUtils.isDryRun(task)) {
+			if (TaskUtil.isDryRun(task)) {
 				 options = SelectorOptions.createCollection(GetOperationOptions.createDoNotDiscovery());
 			} else {
 				options = SelectorOptions.createCollection(GetOperationOptions.createForceRefresh());

diff --git a/...-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/SynchronizationServiceImpl.java b/...-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/SynchronizationServiceImpl.java
@@ -40,6 +40,7 @@
 import com.evolveum.midpoint.schema.util.MiscSchemaUtil;
 import com.evolveum.midpoint.schema.util.ShadowUtil;
 import com.evolveum.midpoint.task.api.Task;
+import com.evolveum.midpoint.task.api.TaskUtil;
 import com.evolveum.midpoint.util.exception.*;
 import com.evolveum.midpoint.util.logging.LoggingUtils;
 import com.evolveum.midpoint.util.logging.Trace;
@@ -420,7 +421,7 @@ private <F extends FocusType> boolean checkProtected(SynchronizationContext<F> s
 	private <F extends FocusType> boolean checkDryRun(SynchronizationContext<F> syncCtx, SynchronizationEventInformation eventInfo, ResourceObjectShadowChangeDescription change, XMLGregorianCalendar now) throws SchemaException {
 		OperationResult subResult = syncCtx.getResult();
 		Task task = syncCtx.getTask();
-		if (ModelImplUtils.isDryRun(task)) {
+		if (TaskUtil.isDryRun(task)) {
 			saveSyncMetadata(syncCtx, change, false, now);
 			subResult.recordSuccess();
 			eventInfo.record(task);

diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/util/ModelImplUtils.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/util/ModelImplUtils.java
@@ -579,57 +579,6 @@ public static void clearRequestee(Task task) {
 	    setRequestee(task, (PrismObject) null);
 	}
 
-	public static boolean isDryRun(Task task) throws SchemaException {
-		Boolean dryRun = findExtensionItemValueInThisOrParent(task, SchemaConstants.MODEL_EXTENSION_DRY_RUN);
-		return dryRun != null ? dryRun : Boolean.FALSE;
-	}
-	private static Boolean findExtensionItemValueInThisOrParent(Task task, QName path) throws SchemaException {
-		Boolean value = findExtensionItemValue(task, path);
-		if (value != null) {
-			return value;
-		}
-		if (task instanceof RunningTask) {
-			RunningTask runningTask = (RunningTask) task;
-			if (runningTask.isLightweightAsynchronousTask() && runningTask.getParentForLightweightAsynchronousTask() != null) {
-				return findExtensionItemValue(runningTask.getParentForLightweightAsynchronousTask(), path);
-			}
-		}
-		return null;
-	}
-
-	private static Boolean findExtensionItemValue(Task task, QName path) throws SchemaException{
-		Validate.notNull(task, "Task must not be null.");
-		if (!task.hasExtension()) {
-			return null;
-		}
-		PrismProperty<Boolean> item = task.getExtensionPropertyOrClone(ItemName.fromQName(path));
-		if (item == null || item.isEmpty()) {
-			return null;
-		}
-		if (item.getValues().size() > 1) {
-			throw new SchemaException("Unexpected number of values for option 'dry run'.");
-		}
-		return item.getValues().iterator().next().getValue();
-	}
-
-	static Boolean findItemValue(RunningTask task, QName path) throws SchemaException{
-		Validate.notNull(task, "Task must not be null.");
-		if (!task.hasExtension()) {
-			return null;
-		}
-		PrismProperty<Boolean> item = task.getExtensionPropertyOrClone(ItemName.fromQName(path));
-		if (item == null || item.isEmpty()) {
-			return null;
-		}
-		if (item.getValues().size() > 1) {
-			throw new SchemaException("Unexpected number of values for option 'dry run'.");
-		}
-		return item.getValues().iterator().next().getValue();
-	}
-
-
-
-
 	public static ModelExecuteOptions getModelExecuteOptions(Task task) throws SchemaException {
 		Validate.notNull(task, "Task must not be null.");
 		if (!task.hasExtension()) {

diff --git a/...test/src/test/java/com/evolveum/midpoint/model/intest/sync/TestLiveSyncTaskMechanics.java b/...test/src/test/java/com/evolveum/midpoint/model/intest/sync/TestLiveSyncTaskMechanics.java
@@ -16,6 +16,7 @@
 import com.evolveum.midpoint.task.api.Task;
 import com.evolveum.midpoint.task.api.TaskExecutionStatus;
 import com.evolveum.midpoint.test.DummyResourceContoller;
+import com.evolveum.midpoint.test.TestResource;
 import com.evolveum.midpoint.util.exception.ObjectNotFoundException;
 import com.evolveum.midpoint.util.exception.SchemaException;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.TaskType;
@@ -75,11 +76,14 @@ public class TestLiveSyncTaskMechanics extends AbstractInitializedModelIntegrati
 	private static final File TASK_ERROR_IMPRECISE_FILE = new File(TEST_DIR, "task-intsync-error-imprecise.xml");
 	private static final String TASK_ERROR_IMPRECISE_OID = "c554ec0f-95c3-40ac-b069-876708d28393";
 
+	private static final TestResource TASK_DRY_RUN = new TestResource(TEST_DIR, "task-intsync-dry-run.xml", "8b5b3b2d-6ef7-4cc8-8507-42778e0d869f");
+	private static final TestResource TASK_DRY_RUN_WITH_UPDATE = new TestResource(TEST_DIR, "task-intsync-dry-run-with-update.xml", "ebcc7393-e886-40ae-8a9f-dfa72230c658");
+
 	private static final String USER_P = "user-p-";
 	private static final String USER_I = "user-i-";
 
 	private static final int ERROR_ON = 4;
-	private static final int USERS = 1000;
+	private static final int USERS = 100;
 
 	public static long delay = 1;                 // referenced from resource-dummy-interrupted-sync.xml
 	public static String errorOn = null;           // referenced from resource-dummy-interrupted-sync.xml
@@ -122,6 +126,12 @@ public void initSystem(Task initTask, OperationResult initResult) throws Excepti
 		addObject(TASK_ERROR_IMPRECISE_FILE, initTask, initResult, workerThreadsCustomizer());
 		waitForTaskFinish(TASK_ERROR_IMPRECISE_OID, false);
 
+		addObject(TASK_DRY_RUN.file, initTask, initResult, workerThreadsCustomizer());
+		waitForTaskFinish(TASK_DRY_RUN.oid, false);
+
+		addObject(TASK_DRY_RUN_WITH_UPDATE.file, initTask, initResult, workerThreadsCustomizer());
+		waitForTaskFinish(TASK_DRY_RUN_WITH_UPDATE.oid, false);
+
 		assertUsers(getNumberOfUsers());
 		for (int i = 0; i < USERS; i++) {
 			interruptedSyncController.addAccount(getUserName(i, true));
@@ -572,6 +582,85 @@ public void test135ErrorImprecise() throws Exception {
 		}
 	}
 
+	/**
+	 * Dry run. Should process all records, but create no users and not update the token.
+	 */
+	@Test
+	public void test140DryRun() throws Exception {
+		final String TEST_NAME = "test140DryRun";
+		displayTestTitle(TEST_NAME);
+
+		// GIVEN
+		Task task = createTask(AbstractSynchronizationStoryTest.class.getName() + "." + TEST_NAME);
+		OperationResult result = task.getResult();
+
+		ObjectQuery query = getStartsWithQuery(USER_P);
+		deleteUsers(query, result);
+
+		// Changes are provided and processed normally.
+		interruptedSyncController.getDummyResource().setOperationDelayOffset(0);
+		delay = 0;
+		errorOn = null;
+
+		// WHEN
+		displayWhen(TEST_NAME);
+
+		waitForTaskNextRun(TASK_DRY_RUN.oid, false, 10000, true);
+
+		// THEN
+		displayThen(TEST_NAME);
+
+		Task taskAfter = taskManager.getTaskWithResult(TASK_DRY_RUN.oid, result);
+		display("Task after", taskAfter);
+		assertSuccess(taskAfter.getResult());
+		assertTaskClosed(taskAfter);
+
+		Integer token = taskAfter.getExtensionPropertyRealValue(SchemaConstants.SYNC_TOKEN);
+		assertEquals("Wrong token value", (Integer) 0, token);
+
+		assertObjects(UserType.class, query, 0);
+	}
+
+	/**
+	 * Dry run with update. Should process all records, but create no users and then update the token.
+	 */
+	@Test
+	public void test150DryRunWithUpdate() throws Exception {
+		final String TEST_NAME = "test150DryRunWithUpdate";
+		displayTestTitle(TEST_NAME);
+
+		// GIVEN
+		Task task = createTask(AbstractSynchronizationStoryTest.class.getName() + "." + TEST_NAME);
+		OperationResult result = task.getResult();
+
+		ObjectQuery query = getStartsWithQuery(USER_P);
+		deleteUsers(query, result);
+
+		// Changes are provided and processed normally.
+		interruptedSyncController.getDummyResource().setOperationDelayOffset(0);
+		delay = 0;
+		errorOn = null;
+
+		// WHEN
+		displayWhen(TEST_NAME);
+
+		waitForTaskNextRun(TASK_DRY_RUN_WITH_UPDATE.oid, false, 10000, true);
+
+		// THEN
+		displayThen(TEST_NAME);
+
+		Task taskAfter = taskManager.getTaskWithResult(TASK_DRY_RUN_WITH_UPDATE.oid, result);
+		display("Task after", taskAfter);
+		assertSuccess(taskAfter.getResult());
+		assertTaskClosed(taskAfter);
+
+		Integer token = taskAfter.getExtensionPropertyRealValue(SchemaConstants.SYNC_TOKEN);
+		assertEquals("Wrong token value", (Integer) USERS, token);
+
+		assertObjects(UserType.class, query, 0);
+	}
+
+
 	private ObjectQuery getStartsWithQuery(String s) {
 		return prismContext.queryFor(UserType.class)
 				.item(UserType.F_NAME).startsWith(s)

diff --git a/model/model-intest/src/test/resources/sync/task-intsync-dry-run-with-update.xml b/model/model-intest/src/test/resources/sync/task-intsync-dry-run-with-update.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2019 Evolveum and contributors
+  ~
+  ~ This work is dual-licensed under the Apache License 2.0 
+  ~ and European Union Public License. See LICENSE file for details.
+  -->
+
+<task oid="ebcc7393-e886-40ae-8a9f-dfa72230c658"
+	xmlns="http://midpoint.evolveum.com/xml/ns/public/common/common-3"
+	xmlns:ext="http://midpoint.evolveum.com/xml/ns/public/model/extension-3"
+	xmlns:ri="http://midpoint.evolveum.com/xml/ns/public/resource/instance-3">
+
+	<name>Live Sync Interrupted: Dry Run with update</name>
+
+	<extension>
+		<ext:objectclass>ri:AccountObjectClass</ext:objectclass>
+		<ext:dryRun>true</ext:dryRun>
+		<ext:updateLiveSyncTokenInDryRun>true</ext:updateLiveSyncTokenInDryRun>
+	</extension>
+
+	<taskIdentifier>ebcc7393-e886-40ae-8a9f-dfa72230c658</taskIdentifier>
+	<ownerRef oid="00000000-0000-0000-0000-000000000002"/>
+	<executionStatus>runnable</executionStatus>
+
+	<handlerUri>http://midpoint.evolveum.com/xml/ns/public/model/synchronization/task/live-sync/handler-3</handlerUri>
+	<objectRef oid="7a58233a-1cfb-46d1-a404-08cdf4626ebb" type="ResourceType"/>
+	<recurrence>single</recurrence>
+</task>
diff --git a/model/model-intest/src/test/resources/sync/task-intsync-dry-run.xml b/model/model-intest/src/test/resources/sync/task-intsync-dry-run.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2019 Evolveum and contributors
+  ~
+  ~ This work is dual-licensed under the Apache License 2.0 
+  ~ and European Union Public License. See LICENSE file for details.
+  -->
+
+<task oid="8b5b3b2d-6ef7-4cc8-8507-42778e0d869f"
+	xmlns="http://midpoint.evolveum.com/xml/ns/public/common/common-3"
+	xmlns:ext="http://midpoint.evolveum.com/xml/ns/public/model/extension-3"
+	xmlns:ri="http://midpoint.evolveum.com/xml/ns/public/resource/instance-3">
+
+	<name>Live Sync Interrupted: Dry Run</name>
+
+	<extension>
+		<ext:objectclass>ri:AccountObjectClass</ext:objectclass>
+		<ext:dryRun>true</ext:dryRun>
+	</extension>
+
+	<taskIdentifier>8b5b3b2d-6ef7-4cc8-8507-42778e0d869f</taskIdentifier>
+	<ownerRef oid="00000000-0000-0000-0000-000000000002"/>
+	<executionStatus>runnable</executionStatus>
+
+	<handlerUri>http://midpoint.evolveum.com/xml/ns/public/model/synchronization/task/live-sync/handler-3</handlerUri>
+	<objectRef oid="7a58233a-1cfb-46d1-a404-08cdf4626ebb" type="ResourceType"/>
+	<recurrence>single</recurrence>
+</task>
diff --git a/...ing-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/sync/LiveSynchronizer.java b/...ing-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/sync/LiveSynchronizer.java
@@ -23,6 +23,7 @@
 import com.evolveum.midpoint.task.api.RunningTask;
 import com.evolveum.midpoint.task.api.Task;
 import com.evolveum.midpoint.task.api.TaskManager;
+import com.evolveum.midpoint.task.api.TaskUtil;
 import com.evolveum.midpoint.util.Holder;
 import com.evolveum.midpoint.util.exception.*;
 import com.evolveum.midpoint.util.logging.LoggingUtils;
@@ -64,6 +65,9 @@ public SynchronizationOperationResult synchronize(ResourceShadowDiscriminator sh
 
         ProvisioningContext ctx = ctxFactory.create(shadowCoordinates, task, parentResult);
         boolean isSimulate = partition != null && partition.getStage() == ExecutionModeType.SIMULATE;
+        boolean isDryRun = TaskUtil.isDryRun(task);
+        boolean updateTokenInDryRun = TaskUtil.findExtensionItemValueInThisOrParent(task,
+                SchemaConstants.MODEL_EXTENSION_UPDATE_LIVE_SYNC_TOKEN_IN_DRY_RUN, false);
 
         PrismProperty<?> initialToken = getTokenFromTask(task);
         syncResult.setInitialToken(initialToken);
@@ -72,6 +76,9 @@ public SynchronizationOperationResult synchronize(ResourceShadowDiscriminator sh
             // (This is introduced in 4.0.1; it is different from the behaviour up to and including 4.0.) The rational is that
             // there's no point in trying to fetch changes after fetching the current token value. We defer that to next run
             // of the live sync task.
+            //
+            // We intentionally update the token even if we are in dry run mode. Otherwise we could never see any records
+            // (without setting updateLiveSyncTokenInDryRun to true).
             fetchAndRememberCurrentToken(syncResult, isSimulate, ctx, parentResult);
             return syncResult;
         }
@@ -208,6 +215,8 @@ private boolean treatError(int sequentialNumber) {
         PrismProperty<?> tokenToSet;
         if (isSimulate) {
             tokenToSet = null;        // Token should not be updated during simulation.
+        } else if (isDryRun && !updateTokenInDryRun) {
+            tokenToSet = null;
         } else if (!syncResult.isHaltingErrorEncountered() && !syncResult.isSuspendEncountered() && syncResult.isAllChangesFetched()) {
             // Everything went OK. Everything was processed.
             PrismProperty<?> finalToken = finalTokenHolder.getValue();