Integrate ID Match to SynchronizationService

1. Correlation state is now stored in the shadows.
2. ID Match-based story test was revived to test at least the basic
functionality.

infra/common/src/main/java/com/evolveum/midpoint/common/SynchronizationUtils.java

@@ -13,6 +13,7 @@
 import javax.xml.datatype.XMLGregorianCalendar;
 import javax.xml.namespace.QName;
 
+import com.evolveum.midpoint.prism.delta.ItemDelta;
 import com.evolveum.midpoint.schema.processor.ResourceObjectTypeDefinition;
 
 import com.evolveum.midpoint.schema.processor.ResourceObjectDefinition;
@@ -63,11 +64,11 @@ private static PropertyDelta<XMLGregorianCalendar> createSynchronizationTimestam
                 .createReplaceDelta(object.getDefinition(), propName, timestamp);
     }
 
-    public static List<PropertyDelta<?>> createSynchronizationSituationAndDescriptionDelta(PrismObject<ShadowType> shadow,
+    public static List<ItemDelta<?, ?>> createSynchronizationSituationAndDescriptionDelta(PrismObject<ShadowType> shadow,
             SynchronizationSituationType situation, String sourceChannel, boolean full, XMLGregorianCalendar timestamp)
             throws SchemaException {
 
-        List<PropertyDelta<?>> propertyDeltas = new ArrayList<>();
+        List<ItemDelta<?, ?>> propertyDeltas = new ArrayList<>();
 
         propertyDeltas.add(
                 createSynchronizationSituationDescriptionDelta(shadow, situation, timestamp, sourceChannel, full));

infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd

@@ -8905,7 +8905,7 @@
                     </xsd:appinfo>
                 </xsd:annotation>
             </xsd:element>
-            <xsd:element name="correlationDefinition" type="tns:CorrelatorsType" minOccurs="0"> <!-- TODO name? -->
+            <xsd:element name="correlators" type="tns:CorrelatorsType" minOccurs="0"> <!-- TODO name? -->
                 <xsd:annotation>
                     <xsd:documentation>
                         Definition of (complex) correlation. TODO
@@ -9715,8 +9715,8 @@
                                 <a:emphasized>true</a:emphasized>
                             </xsd:appinfo>
                         </xsd:annotation>
-                     </xsd:element>
-                     <xsd:element name="pendingOperation" type="tns:PendingOperationType" minOccurs="0" maxOccurs="unbounded">
+                    </xsd:element>
+                    <xsd:element name="pendingOperation" type="tns:PendingOperationType" minOccurs="0" maxOccurs="unbounded">
                         <xsd:annotation>
                             <xsd:documentation>
                                 Description of an operation which which execution is pending for this shadow.
@@ -9727,29 +9727,29 @@
                                 we have not yet received the feedback. Or operations that are executed but are
                                 still kept in the shadow for review by midPoint administrator.
                             </xsd:documentation>
-                             <xsd:appinfo>
+                            <xsd:appinfo>
                                 <a:displayName>ShadowType.pendingOperation</a:displayName>
                                 <a:since>3.6</a:since>
                                 <a:index>presence</a:index>
                             </xsd:appinfo>
                         </xsd:annotation>
-                     </xsd:element>
-                     <xsd:element name="synchronizationSituation" type="tns:SynchronizationSituationType" minOccurs="0">
-                         <xsd:annotation>
+                    </xsd:element>
+                    <xsd:element name="synchronizationSituation" type="tns:SynchronizationSituationType" minOccurs="0">
+                        <xsd:annotation>
                             <xsd:documentation>
                                 The most recent synchronization situation. Regardless of the channel.
                                 This simple enumeration property is used for (indexed) search of shadows in certain situation.
                                 It does not provide any further details. More details can be found in the "synchronizationSituationDescription"
                                 property.
                             </xsd:documentation>
-                             <xsd:appinfo>
+                            <xsd:appinfo>
                                 <a:displayName>ShadowType.synchronizationSituation</a:displayName>
                                 <a:emphasized>true</a:emphasized>
                             </xsd:appinfo>
                         </xsd:annotation>
-                     </xsd:element>
-                     <xsd:element name="synchronizationTimestamp" type="xsd:dateTime" minOccurs="0">
-                         <xsd:annotation>
+                    </xsd:element>
+                    <xsd:element name="synchronizationTimestamp" type="xsd:dateTime" minOccurs="0">
+                        <xsd:annotation>
                             <xsd:documentation>
                                 The timestamp of the most recent synchronization. Regardless of the channel or result.
                                 It does not provide any further details. More details can be found in the "synchronizationSituationDescription"
@@ -9759,25 +9759,38 @@
                                  <a:displayName>ShadowType.synchronizationTimestamp</a:displayName>
                              </xsd:appinfo>
                         </xsd:annotation>
-                     </xsd:element>
-                     <xsd:element name="fullSynchronizationTimestamp" type="xsd:dateTime" minOccurs="0">
-                         <xsd:annotation>
+                    </xsd:element>
+                    <xsd:element name="fullSynchronizationTimestamp" type="xsd:dateTime" minOccurs="0">
+                        <xsd:annotation>
                             <xsd:documentation>
                                 The timestamp of the most recent full synchronization. Regardless of the channel or result.
                                 It does not provide any further details. More details can be found in the "synchronizationSituationDescription"
                                 property.
                             </xsd:documentation>
                         </xsd:annotation>
-                     </xsd:element>
-                     <xsd:element name="synchronizationSituationDescription" type="tns:SynchronizationSituationDescriptionType" minOccurs="0" maxOccurs="unbounded">
-                         <xsd:annotation>
+                    </xsd:element>
+                    <xsd:element name="synchronizationSituationDescription" type="tns:SynchronizationSituationDescriptionType" minOccurs="0" maxOccurs="unbounded">
+                        <xsd:annotation>
                             <xsd:documentation>
                                 Complex description of synchronization situations. This is multi-valued property and it should
                                 store only a single value for each channel (the most recent one). All the older values should be
                                 removed on each update.
                             </xsd:documentation>
                         </xsd:annotation>
-                     </xsd:element>
+                    </xsd:element>
+                    <xsd:element name="correlationState" type="tns:AbstractCorrelationStateType" minOccurs="0">
+                        <xsd:annotation>
+                            <xsd:documentation>
+                                Current state of the correlation process.
+                                This is a correlator-specific structure. As of today it is supported only by ID Match correlator.
+                            </xsd:documentation>
+                            <xsd:appinfo>
+                                <a:since>4.5</a:since>
+                                <a:container>true</a:container>
+                                <a:experimental>true</a:experimental>
+                            </xsd:appinfo>
+                        </xsd:annotation>
+                    </xsd:element>
                     <xsd:element name="objectClass" type="xsd:QName" minOccurs="0">
                         <xsd:annotation>
                             <xsd:documentation>

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/correlator/idmatch/IdMatchService.java → model/model-api/src/main/java/com/evolveum/midpoint/model/api/correlator/idmatch/IdMatchService.java

@@ -1,11 +1,11 @@
 /*
- * Copyright (C) 2010-2021 Evolveum and contributors
+ * Copyright (C) 2010-2022 Evolveum and contributors
  *
  * This work is dual-licensed under the Apache License 2.0
  * and European Union Public License. See LICENSE file for details.
  */
 
-package com.evolveum.midpoint.model.impl.correlator.idmatch;
+package com.evolveum.midpoint.model.api.correlator.idmatch;
 
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowAttributesType;
@@ -15,7 +15,7 @@
 
 /**
  * Represents external ID Match service that is invoked as part of correlation (and other processes)
- * within {@link IdMatchCorrelator}.
+ * within ID Match-based correlator.
  */
 public interface IdMatchService {
 

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/correlator/idmatch/MatchingResult.java → model/model-api/src/main/java/com/evolveum/midpoint/model/api/correlator/idmatch/MatchingResult.java

@@ -1,11 +1,11 @@
 /*
- * Copyright (C) 2010-2021 Evolveum and contributors
+ * Copyright (C) 2010-2022 Evolveum and contributors
  *
  * This work is dual-licensed under the Apache License 2.0
  * and European Union Public License. See LICENSE file for details.
  */
 
-package com.evolveum.midpoint.model.impl.correlator.idmatch;
+package com.evolveum.midpoint.model.api.correlator.idmatch;
 
 import com.evolveum.midpoint.util.DebugDumpable;
 

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/correlator/idmatch/PotentialMatch.java → model/model-api/src/main/java/com/evolveum/midpoint/model/api/correlator/idmatch/PotentialMatch.java

@@ -1,11 +1,11 @@
 /*
- * Copyright (C) 2010-2021 Evolveum and contributors
+ * Copyright (C) 2010-2022 Evolveum and contributors
  *
  * This work is dual-licensed under the Apache License 2.0
  * and European Union Public License. See LICENSE file for details.
  */
 
-package com.evolveum.midpoint.model.impl.correlator.idmatch;
+package com.evolveum.midpoint.model.api.correlator.idmatch;
 
 import com.evolveum.midpoint.util.DebugDumpable;
 import com.evolveum.midpoint.util.DebugUtil;

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/correlator/idmatch/IdMatchCorrelator.java

@@ -7,11 +7,10 @@
 
 package com.evolveum.midpoint.model.impl.correlator.idmatch;
 
-import com.evolveum.midpoint.model.api.correlator.CorrelationContext;
-import com.evolveum.midpoint.model.api.correlator.CorrelationResult;
-import com.evolveum.midpoint.model.api.correlator.Correlator;
+import com.evolveum.midpoint.model.api.correlator.*;
+import com.evolveum.midpoint.model.api.correlator.idmatch.IdMatchService;
+import com.evolveum.midpoint.model.api.correlator.idmatch.MatchingResult;
 import com.evolveum.midpoint.model.impl.ModelBeans;
-import com.evolveum.midpoint.model.api.correlator.CorrelatorConfiguration;
 import com.evolveum.midpoint.model.impl.correlator.CorrelatorUtil;
 import com.evolveum.midpoint.prism.PrismContext;
 import com.evolveum.midpoint.schema.result.OperationResult;

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/correlator/idmatch/IdMatchCorrelatorFactory.java

@@ -9,6 +9,7 @@
 
 import com.evolveum.midpoint.model.api.correlator.CorrelatorFactory;
 import com.evolveum.midpoint.model.api.correlator.CorrelatorFactoryRegistry;
+import com.evolveum.midpoint.model.api.correlator.idmatch.IdMatchService;
 import com.evolveum.midpoint.model.impl.ModelBeans;
 import com.evolveum.midpoint.schema.SchemaConstantsGenerated;
 import com.evolveum.midpoint.schema.result.OperationResult;

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/correlator/idmatch/IdMatchServiceImpl.java

@@ -7,6 +7,8 @@
 
 package com.evolveum.midpoint.model.impl.correlator.idmatch;
 
+import com.evolveum.midpoint.model.api.correlator.idmatch.IdMatchService;
+import com.evolveum.midpoint.model.api.correlator.idmatch.MatchingResult;
 import com.evolveum.midpoint.prism.Item;
 import com.evolveum.midpoint.prism.PrismContainerValue;
 import com.evolveum.midpoint.prism.PrismContext;
@@ -19,15 +21,12 @@
 import com.evolveum.midpoint.xml.ns._public.common.common_3.IdMatchCorrelatorType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowAttributesType;
 
-import com.evolveum.midpoint.xml.ns._public.common.common_3.UserType;
-import com.evolveum.prism.xml.ns._public.types_3.PolyStringType;
 import com.evolveum.prism.xml.ns._public.types_3.ProtectedStringType;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.Collection;
 import java.util.Set;
 
 public class IdMatchServiceImpl implements IdMatchService {

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/executor/LinkUpdater.java

@@ -17,8 +17,8 @@
 import com.evolveum.midpoint.prism.PrismContext;
 import com.evolveum.midpoint.prism.PrismObject;
 import com.evolveum.midpoint.prism.delta.ChangeType;
+import com.evolveum.midpoint.prism.delta.ItemDelta;
 import com.evolveum.midpoint.prism.delta.ObjectDelta;
-import com.evolveum.midpoint.prism.delta.PropertyDelta;
 import com.evolveum.midpoint.prism.util.CloneUtil;
 import com.evolveum.midpoint.provisioning.api.ProvisioningOperationOptions;
 import com.evolveum.midpoint.provisioning.api.ProvisioningService;
@@ -424,7 +424,7 @@ private void updateSituationInShadow(SynchronizationSituationType newSituation,
             }
 
             XMLGregorianCalendar now = clock.currentTimeXMLGregorianCalendar();
-            List<PropertyDelta<?>> syncSituationDeltas = SynchronizationUtils
+            List<ItemDelta<?, ?>> syncSituationDeltas = SynchronizationUtils
                     .createSynchronizationSituationAndDescriptionDelta(currentShadow, newSituation, task.getChannel(),
                             projCtx.hasFullShadow() && TaskUtil.isExecute(task), now);
 

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/SynchronizationContext.java

@@ -9,6 +9,7 @@
 import java.util.List;
 import javax.xml.namespace.QName;
 
+import com.evolveum.midpoint.model.api.correlator.CorrelationContext;
 import com.evolveum.midpoint.prism.util.CloneUtil;
 import com.evolveum.midpoint.schema.processor.ResourceObjectTypeDefinition;
 import com.evolveum.midpoint.model.impl.ModelBeans;
@@ -93,6 +94,12 @@ public class SynchronizationContext<F extends FocusType> implements DebugDumpabl
     /** Situation determined by the sorter or the synchronization service. */
     private SynchronizationSituationType situation;
 
+    /**
+     * Correlation context - in case the correlation was run.
+     * For some correlators it contains the correlation state (to be stored in the shadow).
+     */
+    private CorrelationContext correlationContext;
+
     private String intent;
     private String tag;
 
@@ -162,6 +169,14 @@ public String getIntent() throws SchemaException {
         return intent;
     }
 
+    public CorrelationContext getCorrelationContext() {
+        return correlationContext;
+    }
+
+    public void setCorrelationContext(CorrelationContext correlationContext) {
+        this.correlationContext = correlationContext;
+    }
+
     // TODO reconsider
     public @NotNull ResourceObjectTypeDefinition getObjectTypeDefinition() throws SchemaException, ConfigurationException {
         if (objectTypeDefinition == null) {
@@ -189,8 +204,8 @@ public ExpressionType getConfirmation() {
 
     // TODO reconsider cloning here
     public @NotNull CorrelatorsType getCorrelators() {
-        if (objectSynchronization.getCorrelationDefinition() != null) {
-            return objectSynchronization.getCorrelationDefinition().clone();
+        if (objectSynchronization.getCorrelators() != null) {
+            return objectSynchronization.getCorrelators().clone();
         }
         CorrelatorsType correlators =
                 new CorrelatorsType(PrismContext.get())

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/sync/SynchronizationServiceImpl.java

@@ -17,6 +17,11 @@
 import java.util.List;
 import javax.xml.datatype.XMLGregorianCalendar;
 
+import com.evolveum.midpoint.prism.delta.*;
+import com.evolveum.midpoint.prism.delta.builder.S_ItemEntry;
+
+import com.evolveum.midpoint.prism.util.CloneUtil;
+
 import org.apache.commons.lang.BooleanUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.Validate;
@@ -39,10 +44,6 @@
 import com.evolveum.midpoint.model.impl.lens.*;
 import com.evolveum.midpoint.model.impl.util.ModelImplUtils;
 import com.evolveum.midpoint.prism.*;
-import com.evolveum.midpoint.prism.delta.ChangeType;
-import com.evolveum.midpoint.prism.delta.ItemDeltaCollectionsUtil;
-import com.evolveum.midpoint.prism.delta.ObjectDelta;
-import com.evolveum.midpoint.prism.delta.PropertyDelta;
 import com.evolveum.midpoint.prism.query.ObjectQuery;
 import com.evolveum.midpoint.provisioning.api.ResourceObjectShadowChangeDescription;
 import com.evolveum.midpoint.repo.api.RepositoryService;
@@ -578,6 +579,8 @@ private <F extends FocusType> CorrelationResult correlate(SynchronizationContext
 
         Task task = syncCtx.getTask();
 
+        syncCtx.setCorrelationContext(correlationContext);
+
         return correlatorFactoryRegistry.instantiateCorrelator(syncCtx.getCorrelators(), task, result)
                 .correlate(syncCtx.getShadowedResourceObject().asObjectable(), correlationContext, task, result);
     }
@@ -835,27 +838,32 @@ private <F extends FocusType> void saveSyncMetadata(SynchronizationContext<F> sy
             ShadowType shadowBean = shadow.asObjectable();
             // new situation description
             XMLGregorianCalendar now = clock.currentTimeXMLGregorianCalendar();
-            List<PropertyDelta<?>> deltas = createSynchronizationSituationAndDescriptionDelta(shadow, syncCtx.getSituation(),
-                            change.getSourceChannel(), full, now);
+            List<ItemDelta<?, ?>> deltas =
+                    createSynchronizationSituationAndDescriptionDelta(
+                            shadow, syncCtx.getSituation(), change.getSourceChannel(), full, now);
+
+            S_ItemEntry builder = prismContext.deltaFor(ShadowType.class);
+
+            // TODO revisit this
+            if (syncCtx.getCorrelationContext() != null) {
+                builder = builder.item(ShadowType.F_CORRELATION_STATE).replace(
+                        CloneUtil.clone(syncCtx.getCorrelationContext().getCorrelationState()));
+            }
 
             if (ShadowUtil.isNotKnown(shadowBean.getKind())) {
-                PropertyDelta<ShadowKindType> kindDelta = prismContext.deltaFactory().property().createReplaceDelta(shadow.getDefinition(),
-                        ShadowType.F_KIND, syncCtx.getKind());
-                deltas.add(kindDelta);
+                builder = builder.item(ShadowType.F_KIND).replace(syncCtx.getKind());
             }
 
             if (shouldSaveIntent(syncCtx)) {
-                PropertyDelta<String> intentDelta = prismContext.deltaFactory().property().createReplaceDelta(shadow.getDefinition(),
-                        ShadowType.F_INTENT, syncCtx.getIntent());
-                deltas.add(intentDelta);
+                builder = builder.item(ShadowType.F_INTENT).replace(syncCtx.getIntent());
             }
 
             if (shadowBean.getTag() == null && syncCtx.getTag() != null) {
-                PropertyDelta<String> tagDelta = prismContext.deltaFactory().property().createReplaceDelta(shadow.getDefinition(),
-                        ShadowType.F_TAG, syncCtx.getTag());
-                deltas.add(tagDelta);
+                builder = builder.item(ShadowType.F_TAG).replace(syncCtx.getTag());
             }
 
+            deltas.addAll(builder.asItemDeltas());
+
             repositoryService.modifyObject(shadowBean.getClass(), shadow.getOid(), deltas, result);
             ItemDeltaCollectionsUtil.applyTo(deltas, shadow);
             task.recordObjectActionExecuted(shadow, ChangeType.MODIFY, null);

model/model-impl/src/test/java/com/evolveum/midpoint/model/impl/correlator/correlation/TestCorrelators.java

@@ -13,7 +13,7 @@
 import com.evolveum.midpoint.model.api.correlator.CorrelatorFactoryRegistry;
 import com.evolveum.midpoint.model.impl.AbstractInternalModelIntegrationTest;
 import com.evolveum.midpoint.model.impl.correlator.CorrelatorTestUtil;
-import com.evolveum.midpoint.model.impl.correlator.DummyIdMatchServiceImpl;
+import com.evolveum.midpoint.model.test.idmatch.DummyIdMatchServiceImpl;
 import com.evolveum.midpoint.model.impl.correlator.idmatch.IdMatchCorrelatorFactory;
 import com.evolveum.midpoint.schema.constants.SchemaConstants;
 import com.evolveum.midpoint.schema.processor.ResourceObjectTypeDefinition;