Start working on classification simulation

This commit opens the possibility to simulate even low-level shadow
management operations (classification, correlation) that would be
carried out even under "regular" simulation.

In particular, the support for classification simulation was added
at the lowest levels (tasks, provisioning).

Work in progress. Experimental.

infra/schema/src/main/java/com/evolveum/midpoint/schema/TaskExecutionMode.java

@@ -13,6 +13,8 @@
 
 import java.io.Serializable;
 
+import static com.evolveum.midpoint.schema.TaskExecutionMode.TaskPersistenceMode.*;
+
 /**
  * Describes the execution mode this task runs in. For example, if it is a "full execution" or a preview/simulation.
  * Or, if we should work with the production or development configuration.
@@ -22,33 +24,45 @@
 public class TaskExecutionMode implements Serializable {
 
     public static final TaskExecutionMode PRODUCTION =
-            new TaskExecutionMode("PRODUCTION", true, true);
+            new TaskExecutionMode("PRODUCTION", FULL, true);
     public static final TaskExecutionMode SIMULATED_PRODUCTION =
-            new TaskExecutionMode("SIMULATED_PRODUCTION", false, true);
+            new TaskExecutionMode("SIMULATED_PRODUCTION", SHADOWS, true);
     public static final TaskExecutionMode SIMULATED_DEVELOPMENT =
-            new TaskExecutionMode("SIMULATED_DEVELOPMENT", false, false);
+            new TaskExecutionMode("SIMULATED_DEVELOPMENT", SHADOWS, false);
+    public static final TaskExecutionMode SIMULATED_SHADOWS_PRODUCTION =
+            new TaskExecutionMode("SIMULATED_SHADOWS_PRODUCTION", NONE, true);
+    public static final TaskExecutionMode SIMULATED_SHADOWS_DEVELOPMENT =
+            new TaskExecutionMode("SIMULATED_SHADOWS_DEVELOPMENT", NONE, false);
 
     private final String name;
 
     /** Should the effects of the task be persistent? The value is `false` for preview/simulation. */
-    private final boolean persistent;
+    @NotNull private final TaskPersistenceMode persistenceMode;
 
     /** Should the production configuration be used? */
     private final boolean productionConfiguration;
 
-    private TaskExecutionMode(String name, boolean persistent, boolean productionConfiguration) {
+    private TaskExecutionMode(String name, @NotNull TaskPersistenceMode persistenceMode, boolean productionConfiguration) {
         this.name = name;
-        this.persistent = persistent;
+        this.persistenceMode = persistenceMode;
         this.productionConfiguration = productionConfiguration;
     }
 
-    /** Should the effects of this task be persistent or not? The latter means "simulation", "preview", etc. */
-    public boolean isPersistent() {
-        return persistent;
+    /** TODO */
+    public boolean isFullyPersistent() {
+        return persistenceMode == FULL;
+    }
+
+    public boolean isShadowLevelPersistent() {
+        return persistenceMode == SHADOWS;
+    }
+
+    public boolean isNothingPersistent() {
+        return persistenceMode == NONE;
     }
 
-    public boolean isSimulation() {
-        return !persistent;
+    public boolean areShadowChangesSimulated() {
+        return persistenceMode == NONE;
     }
 
     /**
@@ -60,7 +74,7 @@ public boolean isSimulation() {
      * However, in the future we may provide more customization options here (e.g. explicit enumeration of lifecycle states
      * to use, or even a set of specific deltas to apply).
      *
-     * If {@link #persistent} is `true` then {@link #productionConfiguration} should be `true` as well.
+     * If {@link #persistenceMode} is `true` then {@link #productionConfiguration} should be `true` as well.
      *
      * See https://docs.evolveum.com/midpoint/devel/design/simulations/ for more information.
      */
@@ -77,4 +91,25 @@ public String toString() {
         return new ConfigurationSpecificationType()
                 .productionConfiguration(productionConfiguration);
     }
+
+    enum TaskPersistenceMode {
+        /** Every change is persistent. This is the traditional midPoint operation. */
+        FULL,
+
+        /**
+         * Only changes at the level of shadows are persistent: shadow kind/intent/tag and correlation state.
+         * This is OK for regular "model-level" simulations that can be run after shadow classification and correlation
+         * configuration is fine-tuned.
+         *
+         * TODO specify more precisely
+         */
+        SHADOWS,
+
+        /**
+         * Only changes of very basic shadow operational items (like cached attributes) are persistent.
+         *
+         * TODO specify more precisely
+         */
+        NONE
+    }
 }

infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd

@@ -15685,6 +15685,20 @@
                     </xsd:appinfo>
                 </xsd:annotation>
             </xsd:enumeration>
+            <xsd:enumeration value="00000000-0000-0000-0000-000000000736">
+                <xsd:annotation>
+                    <xsd:appinfo>
+                        <jaxb:typesafeEnumMember name="MARK_SHADOW_CLASSIFICATION_CHANGED"/>
+                    </xsd:appinfo>
+                </xsd:annotation>
+            </xsd:enumeration>
+            <xsd:enumeration value="00000000-0000-0000-0000-000000000737">
+                <xsd:annotation>
+                    <xsd:appinfo>
+                        <jaxb:typesafeEnumMember name="MARK_SHADOW_CORRELATION_STATE_CHANGED"/>
+                    </xsd:appinfo>
+                </xsd:annotation>
+            </xsd:enumeration>
             <xsd:enumeration value="00000000-0000-0000-0000-000000001005">
                 <xsd:annotation>
                     <xsd:appinfo>

infra/schema/src/main/resources/xml/ns/public/common/common-tasks-3.xsd

@@ -2536,13 +2536,28 @@
                             In the future midPoint may also store the computed changes for later analysis.
 
                             TODO we use also the term "simulation"
+
+                            corresponds to TaskExecutionMode.persistenceMode = SHADOWS
                         </p>
                     </xsd:documentation>
                     <xsd:appinfo>
                         <jaxb:typesafeEnumMember name="PREVIEW"/>
                     </xsd:appinfo>
                 </xsd:annotation>
             </xsd:enumeration>
+            <xsd:enumeration value="shadowManagementPreview">
+                <xsd:annotation>
+                    <xsd:documentation>
+                        <p>
+                            TODO ... preview of shadow classification or correlation
+                            corresponds to TaskExecutionMode.persistenceMode = NONE
+                        </p>
+                    </xsd:documentation>
+                    <xsd:appinfo>
+                        <jaxb:typesafeEnumMember name="SHADOW_MANAGEMENT_PREVIEW"/>
+                    </xsd:appinfo>
+                </xsd:annotation>
+            </xsd:enumeration>
             <xsd:enumeration value="dryRun">
                 <xsd:annotation>
                     <xsd:documentation>

model/model-common/src/main/java/com/evolveum/midpoint/model/common/MarkManager.java

@@ -165,7 +165,7 @@ private boolean isEnabled(@NotNull MarkType mark, @NotNull Task task) {
         if (simulationTransaction != null) {
             return simulationTransaction.getSimulationResult().isEventMarkEnabled(mark);
         }
-        if (!task.isSimulatedExecution()) {
+        if (task.isExecutionFullyPersistent()) {
             return false; // Event marks are currently used only for simulations
         }
         // We have no simulation [result] definition, so no custom inclusion/exclusion of event marks.

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelInteractionServiceImpl.java

@@ -185,7 +185,7 @@ public <F extends ObjectType> ModelContext<F> previewChanges(
             throws SchemaException, PolicyViolationException, ExpressionEvaluationException, ObjectNotFoundException, ObjectAlreadyExistsException, CommunicationException, ConfigurationException, SecurityViolationException {
 
         TaskExecutionMode executionMode = task.getExecutionMode();
-        if (executionMode.isPersistent()) {
+        if (executionMode.isFullyPersistent()) {
             LOGGER.warn("Task {} has 'persistent' execution mode when executing previewChanges, setting to SIMULATED_PRODUCTION",
                     task.getName());
 
@@ -203,7 +203,7 @@ public <F extends ObjectType> ModelContext<F> previewChanges(
         if (options.getSimulationOptions() == null) {
             SimulationOptionsType simulation = new SimulationOptionsType();
 
-            SimulationOptionType option = task.isPersistentExecution() ? SimulationOptionType.UNSAFE : SimulationOptionType.SAFE;
+            SimulationOptionType option = task.isExecutionFullyPersistent() ? SimulationOptionType.UNSAFE : SimulationOptionType.SAFE;
             simulation.setCreateOnDemand(option);
             simulation.setSequence(option);
 

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/Clockwork.java

@@ -172,16 +172,16 @@ <F extends ObjectType> HookOperationMode runWithConflictDetection(LensContext<F>
             }
         } finally {
             operationExecutionRecorder.recordOperationExecutions(context, task, result);
-            writeSimulationData(context, task, result);
+            writeFullSimulationData(context, task, result);
             clockworkConflictResolver.unregisterConflictWatcher(context);
             exitCaches();
             context.reportProgress(new ProgressInformation(CLOCKWORK, EXITING));
         }
     }
 
-    private void writeSimulationData(LensContext<?> context, Task task, OperationResult result) {
+    private void writeFullSimulationData(LensContext<?> context, Task task, OperationResult result) {
         SimulationTransaction transactionContext = task.getSimulationTransaction();
-        if (task.isSimulatedExecution() && transactionContext != null) {
+        if (!task.isExecutionFullyPersistent() && transactionContext != null) {
             transactionContext.writeSimulationData(FullOperationSimulationDataImpl.with(context), task, result);
         }
     }

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/ClockworkAuditHelper.java

@@ -99,7 +99,7 @@ <F extends ObjectType> void auditEvent(
             LensContext<F> context, AuditEventStage stage, XMLGregorianCalendar timestamp,
             boolean alwaysAudit, Task task, OperationResult result, OperationResult overallResult) {
 
-        if (!task.isPersistentExecution()) {
+        if (!task.isExecutionFullyPersistent()) {
             // Or, should we record the simulation deltas here? It is better done at the end, because we have all deltas there,
             // so we can have one aggregated delta per object.
             LOGGER.trace("No persistent execution, no auditing");

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/ElementState.java

@@ -726,7 +726,7 @@ void updateAfterExecution(@NotNull TaskExecutionMode taskExecutionMode, int exec
 
         archivedSecondaryDeltas.add(executionWave, secondaryDelta);
 
-        if (!taskExecutionMode.isPersistent()) {
+        if (!taskExecutionMode.isFullyPersistent()) {
             // FIXME temporary code
             if (currentObject == null && isAdd(primaryDelta)) {
                 currentObject = primaryDelta.getObjectToAdd().clone();

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/LensContext.java

@@ -630,7 +630,7 @@ void setRequestAuthorized(boolean isRequestAuthorized) {
      * Makes the context and all sub-context non-fresh.
      */
     public void rot(String reason) {
-        if (!taskExecutionMode.isPersistent()) {
+        if (!taskExecutionMode.isFullyPersistent()) {
             LOGGER.trace("Not rotting the context ({}) because we are not in persistent execution mode", reason);
             return;
         }
@@ -649,7 +649,7 @@ public void rot(String reason) {
      * This is more intelligent than rot()
      */
     private void rotAfterExecution() throws SchemaException, ConfigurationException {
-        if (!taskExecutionMode.isPersistent()) {
+        if (!taskExecutionMode.isFullyPersistent()) {
             LOGGER.trace("Not rotting the context because we are not in persistent execution mode");
             return;
         }

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/OperationExecutionRecorderForClockwork.java

@@ -75,7 +75,7 @@ class OperationExecutionRecorderForClockwork {
 
     <F extends ObjectType> void recordOperationExecutions(LensContext<F> lensContext, Task task, OperationResult parentResult) {
 
-        if (!task.isPersistentExecution()) {
+        if (!task.isExecutionFullyPersistent()) {
             LOGGER.trace("Skipping operation execution recording, as we are in simulated execution mode");
             return;
         }

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/executor/ChangeModeApplication.java

@@ -86,7 +86,7 @@ private void processObjectAdd() {
         Holder<Boolean> reportOnlySeen = new Holder<>(false);
         PrismObject<E> reduced1 = reduce(objectToAdd, configuration.getIgnoredItems(), ignoredSeen);
         PrismObject<E> reduced2 =
-                task.isPersistentExecution() ?
+                task.isExecutionFullyPersistent() ?
                         reduce(reduced1, configuration.getReportOnlyItems(), reportOnlySeen) :
                         reduced1; // no need to skip "report" items, as they will be provided in the simulation report
 
@@ -132,11 +132,11 @@ private void processObjectModify() {
                     toApply.add(modification);
                     break;
                 case REPORT:
-                    if (task.isSimulatedExecution()) {
+                    if (task.isExecutionFullyPersistent()) {
+                        toReport.add(modification);
+                    } else {
                         // "report" items will be provided in the simulation report
                         toApply.add(modification);
-                    } else {
-                        toReport.add(modification);
                     }
                     break;
                 case IGNORE:

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/executor/DeltaExecution.java

@@ -451,7 +451,7 @@ private void executeAddition(OperationResult result)
             deltaForExecution = processChangeApplicationMode(result);
             objectToAdd = deltaForExecution.getObjectToAdd();
             String oid;
-            if (task.isPersistentExecution()) {
+            if (task.isExecutionFullyPersistent()) {
                 oid = executeRealAddition(objectToAdd, result);
             } else {
                 oid = executeSimulatedAddition(objectToAdd);
@@ -587,7 +587,7 @@ private void executeModification(OperationResult result)
             }
 
             deltaForExecution = processChangeApplicationMode(result);
-            if (task.isPersistentExecution()) {
+            if (task.isExecutionFullyPersistent()) {
                 executeRealModification(objectClass, result);
             }
             task.recordObjectActionExecuted(
@@ -742,7 +742,7 @@ private void executeDeletion(OperationResult result)
                     task,
                     result);
 
-            if (task.isPersistentExecution()) {
+            if (task.isExecutionFullyPersistent()) {
                 executeRealDeletion(objectTypeClass, oid, result);
             }
             deleted = true;

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/executor/LinkUpdater.java

@@ -224,7 +224,7 @@ private void executeFocusDelta(ObjectDelta<F> delta, String opName, OperationRes
         String channel = focusContext.getLensContext().getChannel();
         OperationResult result = parentResult.createSubresult(opName);
         try {
-            boolean real = task.isPersistentExecution();
+            boolean real = task.isExecutionFullyPersistent();
             if (real) {
                 repositoryService.modifyObject(focusType, focus.getOid(), delta.getModifications(), result);
             }
@@ -425,12 +425,12 @@ private void updateSituationInShadow(SynchronizationSituationType newSituation,
 
             XMLGregorianCalendar now = clock.currentTimeXMLGregorianCalendar();
             // We consider the shadow to be fully synchronized, as we are processing its owner in the clockwork now.
-            boolean fullSynchronization = projCtx.hasFullShadow() && task.isPersistentExecution();
+            boolean fullSynchronization = projCtx.hasFullShadow() && task.isExecutionFullyPersistent();
             List<ItemDelta<?, ?>> syncSituationDeltas =
                     SynchronizationUtils.createSynchronizationSituationAndDescriptionDelta(
                             currentShadow.asObjectable(), newSituation, task.getChannel(), fullSynchronization, now);
 
-            boolean real = task.isPersistentExecution();
+            boolean real = task.isExecutionFullyPersistent();
             if (real) {
                 executeShadowDelta(syncSituationDeltas, result);
             }

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/executor/ScriptExecutor.java

@@ -92,7 +92,7 @@ void executeReconciliationScripts(BeforeAfterType order, OperationResult result)
             return;
         }
 
-        if (!task.isPersistentExecution()) {
+        if (!task.isExecutionFullyPersistent()) {
             LOGGER.trace("Not a persistent execution. Skipping processing reconciliation scripts.");
             return;
         }

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/ShadowUpdater.java

@@ -62,7 +62,7 @@ ShadowUpdater updateAllSyncMetadataRespectingMode() throws SchemaException {
 
         XMLGregorianCalendar now = beans.clock.currentTimeXMLGregorianCalendar();
 
-        if (syncCtx.isPersistentExecution()) {
+        if (syncCtx.isExecutionFullyPersistent()) {
             updateSyncSituation();
             updateSyncSituationDescription(now);
             updateBasicSyncTimestamp(now); // this is questionable, but the same behavior is in LinkUpdater class

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/SynchronizationContext.java

@@ -483,8 +483,8 @@ void recordSyncStartInTask() {
 
     public abstract boolean isComplete();
 
-    boolean isPersistentExecution() {
-        return task.isPersistentExecution();
+    boolean isExecutionFullyPersistent() {
+        return task.isExecutionFullyPersistent();
     }
 
     public boolean isVisible() {

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/SynchronizationServiceImpl.java

@@ -110,7 +110,7 @@ public void notifyChange(
                 synchronizationFailure = false;
             }
 
-            if (completeCtx.isPersistentExecution() && !synchronizationFailure) {
+            if (completeCtx.isExecutionFullyPersistent() && !synchronizationFailure) {
                 completeCtx.getUpdater()
                         .updateFullSyncTimestamp(clock.currentTimeXMLGregorianCalendar())
                         .commit(result);

model/model-test/src/main/java/com/evolveum/midpoint/model/test/util/SynchronizationRequest.java

@@ -112,7 +112,7 @@ public String execute(OperationResult result) throws CommonException {
                                                 .productionConfiguration(
                                                         taskExecutionMode.isProductionConfiguration()))
                                 .createSimulationResult(
-                                        test.isNativeRepository() && !taskExecutionMode.isPersistent())));
+                                        test.isNativeRepository() && !taskExecutionMode.isFullyPersistent())));
         String taskOid = test.addObject(importTask, task, result);
         if (tracingProfile != null) {
             test.traced(
@@ -131,7 +131,7 @@ public String execute(OperationResult result) throws CommonException {
     }
 
     private @NotNull ExecutionModeType getBackgroundTaskExecutionMode() {
-        if (taskExecutionMode.isPersistent()) {
+        if (taskExecutionMode.isFullyPersistent()) {
             return ExecutionModeType.FULL;
         } else {
             return ExecutionModeType.PREVIEW;
@@ -199,7 +199,6 @@ private void executeImportOnForeground(OperationResult result, String shadowOid)
     @SuppressWarnings("WeakerAccess")
     public TestSimulationResult executeOnForegroundSimulated(
             SimulationDefinitionType simulationDefinition, Task task, OperationResult result) throws CommonException {
-        stateCheck(taskExecutionMode.isSimulation(), "No simulation? Mode = %s", taskExecutionMode);
         return test.executeWithSimulationResult(
                 taskExecutionMode,
                 simulationDefinition,