Add "useReturnedValuesAsBase" correlation option

This option is intended to experiment with ID Match correlation
in midPoint 4.5-M1.

gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/impl/page/admin/cases/component/PotentialMatchDto.java

@@ -50,6 +50,13 @@ public Serializable getAttributeValue(ItemPath path) {
         return value;
     }
 
+    /**
+     * This method falls back to case-insensitive search. This is needed if we try to compare
+     * midPoint shadow with options returned from ID Match (a kind of apples-vs-oranges situation,
+     * because COmanage Match does not support uppercase characters in attribute names). But when midPoint
+     * is switched to provide ID Match-returned value for the basis of comparison, then the case
+     * ignorance is not necessary.
+     */
     private Item findMatchingItem(ItemPath path) {
         Item exactMatch = shadowAttributesType.asPrismContainerValue().findItem(path);
         if (exactMatch != null) {

infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd

@@ -23439,6 +23439,17 @@
                             </xsd:documentation>
                         </xsd:annotation>
                     </xsd:element>
+                    <xsd:element name="useReturnedValuesAsBase" type="xsd:boolean" minOccurs="0" default="false">
+                        <xsd:annotation>
+                            <xsd:documentation>
+                                If true, then values returned from ID Match service (i.e. name-normalized
+                                and filtered) are presented to the user as attribute values of the account that is to
+                                be correlated. If false (the default), values from midPoint (i.e. original ones) are used.
+
+                                This option is here just to experiment with various approaches in midPoint 4.5-M1.
+                            </xsd:documentation>
+                        </xsd:annotation>
+                    </xsd:element>
                     <xsd:element name="url" type="xsd:string" minOccurs="0">
                         <xsd:annotation>
                             <xsd:documentation>

model/model-api/src/main/java/com/evolveum/midpoint/model/api/correlator/idmatch/MatchingResult.java

@@ -38,6 +38,10 @@ public class MatchingResult implements DebugDumpable {
 
     /**
      * A list of potential matches (non-empty when uncertain situation occurred, empty otherwise).
+     *
+     * Note that "create new identity" may or may not be present among the matches. This depends
+     * solely on the approach taken by ID Match service used. MidPoint treats such option according
+     * to its configuration. (At least for now.)
      */
     @NotNull private final Collection<PotentialMatch> potentialMatches;
 

model/model-api/src/main/java/com/evolveum/midpoint/model/api/correlator/idmatch/PotentialMatch.java

@@ -12,6 +12,7 @@
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowAttributesType;
 
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.util.Objects;
 
@@ -26,16 +27,16 @@ public class PotentialMatch implements DebugDumpable {
     private final Integer confidence;
 
     /**
-     * Reference ID. If a record comes from ID Match without reference ID, it is not included here.
+     * Reference ID. Null value means "create a new identity". (If the service provides such option.)
      */
-    @NotNull private final String referenceId;
+    @Nullable private final String referenceId;
 
     /**
      * Attributes of the identity.
      */
     @NotNull private final ShadowAttributesType attributes;
 
-    public PotentialMatch(Integer confidence, @NotNull String referenceId, @NotNull ShadowAttributesType attributes) {
+    public PotentialMatch(Integer confidence, @Nullable String referenceId, @NotNull ShadowAttributesType attributes) {
         this.confidence = confidence;
         this.referenceId = referenceId;
         this.attributes = attributes;
@@ -45,10 +46,15 @@ public Integer getConfidence() {
         return confidence;
     }
 
-    public @NotNull String getReferenceId() {
+    public @Nullable String getReferenceId() {
         return referenceId;
     }
 
+    /** Is this the option to create a new reference ID (i.e. new identity)? */
+    public boolean isNewIdentity() {
+        return referenceId == null;
+    }
+
     public @NotNull ShadowAttributesType getAttributes() {
         return attributes;
     }
@@ -67,7 +73,7 @@ public boolean equals(Object o) {
         }
         PotentialMatch that = (PotentialMatch) o;
         return Objects.equals(confidence, that.confidence)
-                && referenceId.equals(that.referenceId)
+                && Objects.equals(referenceId, that.referenceId)
                 && attributes.equals(that.attributes);
     }
 

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/correlator/idmatch/IdMatchCorrelator.java

@@ -47,6 +47,7 @@ class IdMatchCorrelator implements Correlator {
     /**
      * Configuration of the this correlator.
      */
+    @SuppressWarnings({ "FieldCanBeLocal", "unused" }) // temporary
     @NotNull private final IdMatchCorrelatorType configuration;
 
     /**
@@ -148,31 +149,40 @@ private CorrelationResult correlateUsingKnownReferenceId(
      * Converts internal {@link MatchingResult} into "externalized" {@link IdMatchCorrelationContextType} bean
      * to be stored in the correlation case.
      *
-     * _Temporarily_ adding also "none of the above" potential match here.
+     * _Temporarily_ adding also "none of the above" potential match here. (If it is not present among options returned
+     * from the ID Match service.)
      */
     private @NotNull IdMatchCorrelationContextType createSpecificCaseContext(
             @NotNull MatchingResult mResult, @NotNull ShadowType resourceObject) {
         IdMatchCorrelationContextType context = new IdMatchCorrelationContextType(PrismContext.get());
+        boolean newIdentityOptionPresent = false;
         for (PotentialMatch potentialMatch : mResult.getPotentialMatches()) {
+            if (potentialMatch.isNewIdentity()) {
+                newIdentityOptionPresent = true;
+            }
             context.getPotentialMatch().add(
-                    createPotentialMatchBeanForExistingId(potentialMatch));
+                    createPotentialMatchBeanFromReturnedMatch(potentialMatch));
+        }
+        if (!newIdentityOptionPresent) {
+            context.getPotentialMatch().add(
+                    createPotentialMatchBeanForNewIdentity(resourceObject));
         }
-        context.getPotentialMatch().add(
-                createPotentialMatchBeanForNewId(resourceObject));
         return context;
     }
 
-    private IdMatchCorrelationPotentialMatchType createPotentialMatchBeanForExistingId(PotentialMatch potentialMatch) {
-        String id = potentialMatch.getReferenceId();
+    private IdMatchCorrelationPotentialMatchType createPotentialMatchBeanFromReturnedMatch(PotentialMatch potentialMatch) {
+        @Nullable String id = potentialMatch.getReferenceId();
+        String optionUri = id != null ?
+                qNameToUri(new QName(SchemaConstants.CORRELATION_NS, SchemaConstants.CORRELATION_OPTION_PREFIX + id)) :
+                SchemaConstants.CORRELATION_NONE_URI;
         return new IdMatchCorrelationPotentialMatchType(PrismContext.get())
-                .uri(qNameToUri(
-                        new QName(SchemaConstants.CORRELATION_NS, SchemaConstants.CORRELATION_OPTION_PREFIX + id)))
+                .uri(optionUri)
                 .confidence(potentialMatch.getConfidence())
                 .referenceId(id)
                 .attributes(potentialMatch.getAttributes());
     }
 
-    private IdMatchCorrelationPotentialMatchType createPotentialMatchBeanForNewId(@NotNull ShadowType resourceObject) {
+    private IdMatchCorrelationPotentialMatchType createPotentialMatchBeanForNewIdentity(@NotNull ShadowType resourceObject) {
         return new IdMatchCorrelationPotentialMatchType(PrismContext.get())
                 .uri(SchemaConstants.CORRELATION_NONE_URI)
                 .attributes(

model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/correlator/idmatch/IdMatchServiceImpl.java

@@ -50,16 +50,25 @@
 import java.util.List;
 import java.util.Objects;
 
+/**
+ * An interface from midPoint to real ID Match service.
+ */
 public class IdMatchServiceImpl implements IdMatchService {
 
+    private static final Trace LOGGER = TraceManager.getTrace(IdMatchServiceImpl.class);
+
     private static final String REFERENCE_REQUEST_ID = "referenceId";
     private static final String MATCH_REQUEST_ID = "matchRequest";
     private static final String CANDIDATES = "candidates";
 
     private static final String MAPPED_ICFS_NAME = "icfs_name"; // temporary
     private static final String MAPPED_ICFS_UID = "icfs_uid"; // temporary
 
-    private static final Trace LOGGER = TraceManager.getTrace(IdMatchServiceImpl.class);
+    /**
+     * Value get from or sent to ID Match Service denoting the fact that a new reference ID is to be or should be created.
+     * Part of ID Match API.
+     */
+    private static final String NEW_REFERENCE_ID = "new";
 
     /** URL where the service resides. */
     @NotNull private final String url;
@@ -78,19 +87,27 @@ public class IdMatchServiceImpl implements IdMatchService {
     /** A shadow attribute to be used as SOR ID in the requests. */
     @NotNull private final QName sorIdAttribute;
 
+    /**
+     * Should we return "none of the above matches" option among potential matches? (Under assumption that
+     * ID Match service provides us with such option?)
+     */
+    private final boolean includeNoneMatchesOption;
+
     private static final QName DEFAULT_SOR_ID_ATTRIBUTE = SchemaConstants.ICFS_UID;
 
     private IdMatchServiceImpl(
             @NotNull String url,
             @Nullable String username,
             @Nullable ProtectedStringType password,
             @Nullable String sorLabel,
-            @Nullable QName sorIdAttribute) {
+            @Nullable QName sorIdAttribute,
+            boolean includeNoneMatchesOption) {
         this.url = url;
         this.username = username;
         this.password = password;
         this.sorLabel = Objects.requireNonNullElse(sorLabel, DEFAULT_SOR_LABEL);
         this.sorIdAttribute = Objects.requireNonNullElse(sorIdAttribute, DEFAULT_SOR_ID_ATTRIBUTE);
+        this.includeNoneMatchesOption = includeNoneMatchesOption;
     }
 
     @Override
@@ -165,6 +182,9 @@ private Client createClient() {
     /**
      * Creates a list of potential matches from the ID Match response.
      *
+     * All options presented by the ID Match service are converted. So if there is an option to create a new identity,
+     * it is included in the returned value as well.
+     *
      * Expected entity (see `entityObject` variable):
      *
      * ----
@@ -203,12 +223,10 @@ private Client createClient() {
 
     private List<PotentialMatch> createPotentialMatches(String entity) throws SchemaException, JSONException {
         LOGGER.info("Creating potential matches from:\n{}", entity);
-        System.out.println("Creating potential matches from: "+ entity);
         List<PotentialMatch> potentialMatches = new ArrayList<>();
 
         JSONObject entityObject = new JSONObject(entity);
         JSONArray candidatesArray = entityObject.getJSONArray(CANDIDATES);
-        System.out.println("Candidates array: " + candidatesArray);
 
         for (int i = 0; i < candidatesArray.length(); i++) {
             potentialMatches.addAll(
@@ -229,10 +247,13 @@ private List<PotentialMatch> createPotentialMatches(String entity) throws Schema
     private List<PotentialMatch> createPotentialMatches(Object candidate) throws SchemaException, JSONException {
         JSONObject jsonObject = MiscUtil.castSafely(candidate, JSONObject.class);
 
-        String referenceId = jsonObject.getString(REFERENCE_REQUEST_ID);
-        if (referenceId == null || referenceId.isEmpty() || referenceId.equals("new")) {
-            return List.of(); // Temporary: "new reference ID" should be NOT included among potential matches
+        String rawReferenceId = jsonObject.getString(REFERENCE_REQUEST_ID);
+        String referenceId = fromRawReferenceId(rawReferenceId);
+        if (referenceId == null && !includeNoneMatchesOption) {
+            LOGGER.trace("Skipping 'none matches' option: {}", candidate);
+            return List.of();
         }
+
         if (jsonObject.has("sorRecords")) {
             List<PotentialMatch> potentialMatches = new ArrayList<>();
             JSONArray sorRecordsArray = jsonObject.getJSONArray("sorRecords");
@@ -251,6 +272,17 @@ private List<PotentialMatch> createPotentialMatches(Object candidate) throws Sch
         }
     }
 
+    /**
+     * Treats empty/"new" reference ID as no reference ID.
+     */
+    private String fromRawReferenceId(String rawReferenceId) {
+        if (rawReferenceId != null && !rawReferenceId.isEmpty() && !rawReferenceId.equals(NEW_REFERENCE_ID)) {
+            return rawReferenceId;
+        } else {
+            return null;
+        }
+    }
+
     private PotentialMatch createPotentialMatchFromSorAttributes(String referenceId, JSONObject sorAttributes)
             throws JSONException, SchemaException {
         ShadowAttributesType attributes = new ShadowAttributesType(PrismContext.get());
@@ -401,7 +433,7 @@ public void resolve(
             @Nullable String referenceId,
             @NotNull OperationResult result) throws CommunicationException, SchemaException {
 
-        String nativeReferenceId = referenceId != null ? referenceId : "new";
+        String nativeReferenceId = referenceId != null ? referenceId : NEW_REFERENCE_ID;
         //noinspection unchecked
         JsonRequest request = generateResolveRequest(
                 attributes.asPrismContainerValue(), nativeReferenceId, matchRequestId);
@@ -419,7 +451,8 @@ public static IdMatchServiceImpl instantiate(@NotNull IdMatchCorrelatorType conf
                 configuration.getUsername(),
                 configuration.getPassword(),
                 configuration.getSorLabel(),
-                configuration.getSorIdentifierAttribute());
+                configuration.getSorIdentifierAttribute(),
+                Boolean.TRUE.equals(configuration.isUseReturnedValuesAsBase())); // default is "false"
     }
 
     /**
@@ -430,6 +463,13 @@ public static IdMatchServiceImpl instantiate(
             @NotNull String url,
             @Nullable String username,
             @Nullable ProtectedStringType password) {
-        return new IdMatchServiceImpl(url, username, password, null, null);
+        return new IdMatchServiceImpl(
+                url,
+                username,
+                password,
+                null,
+                null,
+                false // Our tests currently expect that "none matches" option is not included.
+        );
     }
 }