Remove many uses of task extension items

1) Updated LiveSyncWorkDefinition[Type] to cover all the parameters. 2) Extended TestLiveSyncTask to cover both legacy and new activity spec. 3) Updated provisioning.synchronize(..) method to correctly get all necessary parameters without accessing task extension. 4) Worker threads are now correctly set either by distribution definition or by the legacy task extension. Unrelated changes: - Removed unused items from SchemaConstants.
Evolveum · Jun 30, 2021 · bb2d274 · bb2d274
1 parent 5f254c4
commit bb2d274
Show file tree

Hide file tree

Showing 76 changed files with 1,339 additions and 574 deletions.
diff --git a/...dmin-gui/src/main/java/com/evolveum/midpoint/web/page/admin/server/TaskBasicTabPanel.java b/...dmin-gui/src/main/java/com/evolveum/midpoint/web/page/admin/server/TaskBasicTabPanel.java
@@ -222,7 +222,7 @@ private ItemVisibility getBasicTabVisibility(ItemPath path) {
         } else if (taskHandler.endsWith("task/workers-restart/handler-3")) {
             //no attributes
         } else if (taskHandler.endsWith("model/synchronization/task/delete-not-updated-shadow/handler-3")) {
-            pathsToShow = Arrays.asList(ItemPath.create(TaskType.F_EXTENSION, SchemaConstants.MODEL_EXTENSION_NOT_UPDATED_SHADOW_DURATION),
+            pathsToShow = Arrays.asList(ItemPath.create(TaskType.F_EXTENSION, SchemaConstants.LEGACY_NOT_UPDATED_DURATION_PROPERTY_NAME),
                     ItemPath.create(TaskType.F_EXTENSION, SchemaConstants.MODEL_EXTENSION_WORKER_THREADS),
                     TaskType.F_OBJECT_REF);
         } else if (taskHandler.endsWith("model/shadowRefresh/handler-3")) {

diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/SchemaConstants.java
diff --git a/...ema/src/main/java/com/evolveum/midpoint/schema/util/task/work/ActivityDefinitionUtil.java b/...ema/src/main/java/com/evolveum/midpoint/schema/util/task/work/ActivityDefinitionUtil.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2010-2021 Evolveum and contributors
+ *
+ * This work is dual-licensed under the Apache License 2.0
+ * and European Union Public License. See LICENSE file for details.
+ */
+
+package com.evolveum.midpoint.schema.util.task.work;
+
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ActivityDefinitionType;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.WorkDistributionType;
+
+import org.jetbrains.annotations.NotNull;
+
+public class ActivityDefinitionUtil {
+
+    public static @NotNull WorkDistributionType findOrCreateDistribution(ActivityDefinitionType activity) {
+        if (activity.getDistribution() != null) {
+            return activity.getDistribution();
+        } else {
+            return activity.beginDistribution();
+        }
+    }
+}
diff --git a/infra/schema/src/main/resources/xml/ns/public/common/common-tasks-3.xsd b/infra/schema/src/main/resources/xml/ns/public/common/common-tasks-3.xsd
@@ -6111,7 +6111,25 @@
                     <xsd:element name="batchSize" type="xsd:int" minOccurs="0">
                         <xsd:annotation>
                             <xsd:documentation>
-                                TODO
+                                Batch size for live sync operations: if specified as N > 0, live synchronization fetches
+                                at most N records during each provisioning.synchronize() method execution - i.e. during
+                                any live sync task run. (Currently, it leaves unprocessed changes for the next live sync
+                                task run. This may change in the future.)
+
+                                USE WITH CARE. This feature assumes that the connector provides LiveSync capability
+                                with preciseTokenValue = true, i.e. that it assigns sync tokens to individual changes
+                                "incrementally" so it can resume live sync operation on each individual change. This
+                                can be the case of SQL connectors, but e.g. not for CSV or LDAP ones.
+                            </xsd:documentation>
+                        </xsd:annotation>
+                    </xsd:element>
+                    <xsd:element name="updateLiveSyncTokenInDryRun" type="xsd:boolean" minOccurs="0" default="false">
+                        <xsd:annotation>
+                            <xsd:documentation>
+                                Indicates if the LiveSync token should be updated when running in dry run mode.
+                                If false (the default) then LiveSync will not update token to a new value so it
+                                will process objects fetched on the next run (either dry or normal). If true, it
+                                will update the token, and therefore marks objects as processed.
                             </xsd:documentation>
                         </xsd:annotation>
                     </xsd:element>
@@ -6599,6 +6617,32 @@
     </xsd:complexType>
     <xsd:element name="reconciliationWorkState" type="tns:ReconciliationWorkStateType" />
 
+    <xsd:complexType name="LiveSyncWorkStateType">
+        <xsd:annotation>
+            <xsd:documentation>
+                TODO
+            </xsd:documentation>
+            <xsd:appinfo>
+                <a:container>true</a:container>
+                <a:since>4.4</a:since>
+            </xsd:appinfo>
+        </xsd:annotation>
+        <xsd:complexContent>
+            <xsd:extension base="tns:AbstractActivityWorkStateType">
+                <xsd:sequence>
+                    <xsd:element name="token" type="xsd:anyType" minOccurs="0">
+                        <xsd:annotation>
+                            <xsd:documentation>
+                                The token value.
+                            </xsd:documentation>
+                        </xsd:annotation>
+                    </xsd:element>
+                </xsd:sequence>
+            </xsd:extension>
+        </xsd:complexContent>
+    </xsd:complexType>
+    <xsd:element name="liveSyncWorkState" type="tns:LiveSyncWorkStateType" />
+
     <xsd:complexType name="WorkersReconciliationResultType">
         <xsd:annotation>
             <xsd:documentation>

diff --git a/...ava/com/evolveum/midpoint/model/common/expression/functions/BasicExpressionFunctions.java b/...ava/com/evolveum/midpoint/model/common/expression/functions/BasicExpressionFunctions.java
@@ -1241,6 +1241,9 @@ public static String debugDump(Object o, int indent) {
         return DebugUtil.debugDump(o, indent);
     }
 
+    /**
+     * TODO Prepare also non-legacy version of this method.
+     */
     public void setTaskWorkerThreads(TaskType task, Integer value) throws SchemaException {
         Objects.requireNonNull(task, "task is not specified");
         ObjectTypeUtil.setExtensionPropertyRealValues(prismContext, task.asPrismContainerValue(),

diff --git a/...odel-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/SynchronizationContext.java b/...odel-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/SynchronizationContext.java
@@ -66,7 +66,7 @@ public class SynchronizationContext<F extends FocusType> implements DebugDumpabl
     private String channel;
     private ExpressionProfile expressionProfile;
 
-    private Task task;
+    private final Task task;
 
     private ObjectSynchronizationType objectSynchronization;
     private Class<F> focusClass;
@@ -399,10 +399,6 @@ public Task getTask() {
         return task;
     }
 
-    public void setTask(Task task) {
-        this.task = task;
-    }
-
     boolean isShadowExistsInRepo() {
         return shadowExistsInRepo;
     }

diff --git a/...-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/SynchronizationServiceImpl.java b/...-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/SynchronizationServiceImpl.java
@@ -121,15 +121,16 @@ public void notifyChange(@NotNull ResourceObjectShadowChangeDescription change,
 
             task.onSynchronizationStart(change.getItemProcessingIdentifier(), change.getShadowOid(), syncCtx.getSituation());
 
-            boolean fullRun = !TaskUtil.isDryRun(syncCtx.getTask());
-            saveSyncMetadata(syncCtx, change, fullRun, result);
+            ExecutionModeType executionMode = TaskUtil.getExecutionMode(task);
+            boolean dryRun = executionMode == ExecutionModeType.DRY_RUN;
 
-            if (fullRun) {
+            saveSyncMetadata(syncCtx, change, !dryRun, result);
+
+            if (!dryRun) {
                 reactToChange(syncCtx, change, result);
             }
 
-            LOGGER.debug("SYNCHRONIZATION: DONE ({}) for {}", fullRun ? "full run" : "dry run",
-                    change.getShadowedResourceObject());
+            LOGGER.debug("SYNCHRONIZATION: DONE (mode '{}') for {}", executionMode, change.getShadowedResourceObject());
 
         } catch (SystemException ex) {
             // avoid unnecessary re-wrap

diff --git a/.../com/evolveum/midpoint/model/impl/sync/tasks/recon/RemainingShadowsActivityExecution.java b/.../com/evolveum/midpoint/model/impl/sync/tasks/recon/RemainingShadowsActivityExecution.java
@@ -142,7 +142,7 @@ private void reconcileShadow(PrismObject<ShadowType> shadow, Task task, Operatio
                 shadow.asObjectable().getFullSynchronizationTimestamp());
         try {
             Collection<SelectorOptions<GetOperationOptions>> options;
-            if (TaskUtil.isDryRun(task)) {
+            if (isDryRun()) {
                 options = SelectorOptions.createCollection(GetOperationOptions.createDoNotDiscovery());
             } else {
                 options = SelectorOptions.createCollection(GetOperationOptions.createForceRefresh());

diff --git a/...main/java/com/evolveum/midpoint/model/impl/sync/tasks/sync/LiveSyncActivityExecution.java b/...main/java/com/evolveum/midpoint/model/impl/sync/tasks/sync/LiveSyncActivityExecution.java
@@ -13,8 +13,6 @@
 import com.evolveum.midpoint.schema.result.OperationConstants;
 import com.evolveum.midpoint.schema.result.OperationResultStatus;
 
-import com.evolveum.midpoint.xml.ns._public.common.common_3.ActivityStatePersistenceType;
-
 import com.google.common.annotations.VisibleForTesting;
 import org.jetbrains.annotations.NotNull;
 
@@ -110,9 +108,17 @@ public void allEventsSubmitted(OperationResult result) {
             }
         };
 
+        LiveSyncOptions options = createLiveSyncOptions();
+
         ModelImplUtils.clearRequestee(getRunningTask());
         syncResult = getModelBeans().provisioningService
-                .synchronize(objectClassSpecification.getCoords(), getRunningTask(), isSimulate(), handler, opResult);
+                .synchronize(objectClassSpecification.getCoords(), getRunningTask(), options, handler, opResult);
+    }
+
+    @NotNull
+    private LiveSyncOptions createLiveSyncOptions() {
+        LiveSyncWorkDefinition def = activity.getWorkDefinition();
+        return new LiveSyncOptions(def.getExecutionMode(), def.getBatchSize(), def.isUpdateLiveSyncTokenInDryRun());
     }
 
     @Override
@@ -161,8 +167,10 @@ public boolean providesTracingAndDynamicProfiling() {
     protected @NotNull ErrorHandlingStrategyExecutor.FollowUpAction getDefaultErrorAction() {
         // This could be a bit tricky if combined with partially-specified error handling strategy.
         // So, please, do NOT combine these two! If you specify a strategy, do not use retryLiveSyncErrors extension item.
-        boolean retryErrors = isNotFalse(getRunningTask().getExtensionPropertyRealValue(
-                SchemaConstants.MODEL_EXTENSION_RETRY_LIVE_SYNC_ERRORS)); // FIXME!!!
+        //
+        // TODO remove in the next schema cleanup
+        boolean retryErrors = isNotFalse(
+                getRunningTask().getExtensionPropertyRealValue(SchemaConstants.MODEL_EXTENSION_RETRY_LIVE_SYNC_ERRORS));
         return retryErrors ? STOP : CONTINUE;
     }
 }
diff --git a/...rc/main/java/com/evolveum/midpoint/model/impl/sync/tasks/sync/LiveSyncWorkDefinition.java b/...rc/main/java/com/evolveum/midpoint/model/impl/sync/tasks/sync/LiveSyncWorkDefinition.java
@@ -7,6 +7,8 @@
 
 package com.evolveum.midpoint.model.impl.sync.tasks.sync;
 
+import com.evolveum.midpoint.schema.constants.SchemaConstants;
+
 import org.jetbrains.annotations.NotNull;
 
 import com.evolveum.midpoint.prism.PrismContext;
@@ -23,26 +25,47 @@
 public class LiveSyncWorkDefinition extends AbstractWorkDefinition implements ResourceObjectSetSpecificationProvider {
 
     @NotNull private final ResourceObjectSetType resourceObjects;
+    private final Integer batchSize;
+    private final boolean updateLiveSyncTokenInDryRun;
 
     LiveSyncWorkDefinition(WorkDefinitionSource source) {
+        Boolean updateLiveSyncTokenInDryRunRaw;
         if (source instanceof LegacyWorkDefinitionSource) {
-            resourceObjects = ResourceObjectSetUtil.fromLegacySource((LegacyWorkDefinitionSource) source);
+            LegacyWorkDefinitionSource legacy = (LegacyWorkDefinitionSource) source;
+            resourceObjects = ResourceObjectSetUtil.fromLegacySource(legacy);
+            batchSize = legacy.getExtensionItemRealValue(SchemaConstants.MODEL_EXTENSION_LIVE_SYNC_BATCH_SIZE, Integer.class);
+            updateLiveSyncTokenInDryRunRaw =
+                    legacy.getExtensionItemRealValue(SchemaConstants.MODEL_EXTENSION_UPDATE_LIVE_SYNC_TOKEN_IN_DRY_RUN,
+                            Boolean.class);
         } else {
             LiveSyncWorkDefinitionType typedDefinition = (LiveSyncWorkDefinitionType)
                     ((WorkDefinitionWrapper.TypedWorkDefinitionWrapper) source).getTypedDefinition();
             resourceObjects = typedDefinition.getResourceObjects() != null ?
                     typedDefinition.getResourceObjects() : new ResourceObjectSetType(PrismContext.get());
+            batchSize = typedDefinition.getBatchSize();
+            updateLiveSyncTokenInDryRunRaw = typedDefinition.isUpdateLiveSyncTokenInDryRun();
         }
         ResourceObjectSetUtil.removeQuery(resourceObjects);
+        updateLiveSyncTokenInDryRun = Boolean.TRUE.equals(updateLiveSyncTokenInDryRunRaw);
     }
 
     @Override
     public @NotNull ResourceObjectSetType getResourceObjectSetSpecification() {
         return resourceObjects;
     }
 
+    public Integer getBatchSize() {
+        return batchSize;
+    }
+
+    boolean isUpdateLiveSyncTokenInDryRun() {
+        return updateLiveSyncTokenInDryRun;
+    }
+
     @Override
     protected void debugDumpContent(StringBuilder sb, int indent) {
         DebugUtil.debugDumpWithLabelLn(sb, "resourceObjects", resourceObjects, indent+1);
+        DebugUtil.debugDumpWithLabelLn(sb, "batchSize", batchSize, indent+1);
+        DebugUtil.debugDumpWithLabelLn(sb, "updateLiveSyncTokenInDryRun", updateLiveSyncTokenInDryRun, indent+1);
     }
 }
diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/util/ModelImplUtils.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/util/ModelImplUtils.java
@@ -586,25 +586,6 @@ public static void clearRequestee(Task task) {
         setRequestee(task, (PrismObject) null);
     }
 
-    public static ModelExecuteOptions getModelExecuteOptions(@NotNull Task task) {
-        ModelExecuteOptionsType options1 = task.getExtensionContainerRealValueOrClone(SchemaConstants.C_MODEL_EXECUTE_OPTIONS); // legacy
-        if (options1 != null) {
-            return ModelExecuteOptions.fromModelExecutionOptionsType(options1);
-        }
-
-        ModelExecuteOptionsType options2 = task.getExtensionContainerRealValueOrClone(SchemaConstants.MODEL_EXTENSION_MODEL_EXECUTE_OPTIONS);
-        if (options2 != null) {
-            return ModelExecuteOptions.fromModelExecutionOptionsType(options2);
-        }
-
-        ModelExecuteOptionsType options3 = task.getExtensionContainerRealValueOrClone(SchemaConstants.MODEL_EXTENSION_EXECUTE_OPTIONS);
-        if (options3 != null) {
-            return ModelExecuteOptions.fromModelExecutionOptionsType(options3);
-        }
-
-        return null;
-    }
-
     public static ModelExecuteOptions getModelExecuteOptions(PrismContainerValue<?> taskExtension) {
         if (taskExtension == null) {
             return null;

diff --git a/model/model-impl/src/test/java/com/evolveum/midpoint/model/impl/util/mock/MockFactory.java b/model/model-impl/src/test/java/com/evolveum/midpoint/model/impl/util/mock/MockFactory.java
@@ -8,8 +8,6 @@
 
 import java.util.*;
 
-import com.evolveum.midpoint.util.exception.*;
-
 import org.jetbrains.annotations.NotNull;
 
 import com.evolveum.midpoint.common.refinery.RefinedObjectClassDefinition;
@@ -48,7 +46,7 @@ public <T extends ObjectType> String addObject(PrismObject<T> object, OperationP
 
             @Override
             public @NotNull SynchronizationResult synchronize(ResourceShadowDiscriminator shadowCoordinates, Task task,
-                    boolean simulate, LiveSyncEventHandler handler, OperationResult parentResult) throws ObjectNotFoundException, CommunicationException, SchemaException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException, PolicyViolationException {
+                    LiveSyncOptions options, LiveSyncEventHandler handler, OperationResult parentResult) {
                 return new SynchronizationResult();
             }
 

diff --git a/model/model-impl/src/test/resources/common/task-reconcile-dummy.xml b/model/model-impl/src/test/resources/common/task-reconcile-dummy.xml
@@ -19,6 +19,7 @@
     <!-- Resource printable name -->
     <name>Reconciliation: Dummy</name>
 
+    <!-- Must be legacy spec, because dryRun flag is set in extension by TestReconScript -->
     <extension>
         <syncext:freshnessInterval>1000</syncext:freshnessInterval>
         <syncext:objectclass>ri:AccountObjectClass</syncext:objectclass>

diff --git a/...t/src/test/java/com/evolveum/midpoint/model/intest/AbstractEmptyModelIntegrationTest.java b/...t/src/test/java/com/evolveum/midpoint/model/intest/AbstractEmptyModelIntegrationTest.java
@@ -33,6 +33,7 @@
  * in {@link AbstractConfiguredModelIntegrationTest}.
  *
  * Creates a repo with:
+ *
  *  - empty system configuration
  *  - administrator user (from common dir)
  *  - superuser role (from common dir)

diff --git a/.../src/test/java/com/evolveum/midpoint/model/intest/tasks/DummyInterruptedSyncResource.java b/.../src/test/java/com/evolveum/midpoint/model/intest/tasks/DummyInterruptedSyncResource.java
@@ -44,10 +44,24 @@ public void init(DummyResourceCollection collection, Task task, OperationResult
 
     public static long delay = 1;
     public static String errorOn = null;
+    private static Runnable executionListener;
 
     public void createAccounts(int users, Function<Integer, String> userNameFormatter) throws Exception {
         for (int i = 0; i < users; i++) {
             controller.addAccount(userNameFormatter.apply(i));
         }
     }
+
+    /**
+     * Called from inbound mapping.
+     */
+    public static void onMappingExecution() {
+        if (executionListener != null) {
+            executionListener.run();
+        }
+    }
+
+    public static void setExecutionListener(Runnable executionListener) {
+        DummyInterruptedSyncResource.executionListener = executionListener;
+    }
 }