Merge remote-tracking branch 'origin/master' into tmp/do-not-freeze-e…

…xtensions

* origin/master: (39 commits)
  adding javadoc for method that check presence of HTML format in 'code' xml elemet
  MID-9506: Added schema notion to Axiom Query Visitors
  MID-9506: Added toSearchFilterType to Midpoint Query Serializer
  MID-9461: fix writing of CDATA to xml when html format is used
  Added item definition to axiom query lang service
  Add diagnostic info for MID-9535
  MID-9532: Fixed delta duplicates caused by ConditionalSearchFilterType
  Back to 4.9-SNAPSHOT development version
  Milestone 4.9-M3
  change super class for EncryptionException to CommonException
  Add MiscUtil#stateNonEmpty method
  Support partial impl. of mutable definitions
  Fixed JSON ser/deser issues with assocations
  MID-9413: Query - Fixed incorrect serialiation of Equals PolyString
  secrets provider: added tests for caching, updated documentation, improved environment variables secret provider
  Support new associations
  secret providers: fixing handling of encrypted/external data in protected data type
  secrets providers: support for resolving secrets in BasicExpressionFunctions
  secrets providers: protected data improvements
  docs/comments: more Jira link fixes
  ...

# Conflicts:
#	infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/ComplexTypeDefinitionImpl.java

infra/prism-api/src/main/java/com/evolveum/midpoint/prism/ComplexTypeDefinition.java

@@ -7,10 +7,7 @@
 
 package com.evolveum.midpoint.prism;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
+import java.util.*;
 import javax.xml.namespace.QName;
 
 import org.jetbrains.annotations.NotNull;
@@ -39,7 +36,6 @@ public interface ComplexTypeDefinition extends TypeDefinition, LocalItemDefiniti
      * The list is unmodifiable.
      */
     @Override
-    @SuppressWarnings("unchecked") // temporary workaround
     @NotNull
     List<? extends ItemDefinition<?>> getDefinitions();
 
@@ -88,6 +84,9 @@ public interface ComplexTypeDefinition extends TypeDefinition, LocalItemDefiniti
     @Experimental
     boolean isListMarker();
 
+    /** Type name for items that are not explicitly defined in this CTD. */
+    @Nullable QName getDefaultItemTypeName();
+
     /**
      * When resolving unqualified names for items contained in this CTD, what should be the default namespace
      * to look into at first. Currently does NOT apply recursively (to inner CTDs).
@@ -183,4 +182,14 @@ default boolean isItemDefinitionRemoved(QName itemName) {
     default boolean hasOperationalOnlyItems() {
         return false;
     }
+
+    default List<PrismPropertyDefinition<?>> getPropertyDefinitions() {
+        List<PrismPropertyDefinition<?>> props = new ArrayList<>();
+        for (ItemDefinition<?> def : getDefinitions()) {
+            if (def instanceof PrismPropertyDefinition<?> propertyDefinition) {
+                props.add(propertyDefinition);
+            }
+        }
+        return props;
+    }
 }

infra/prism-api/src/main/java/com/evolveum/midpoint/prism/Definition.java

@@ -65,22 +65,7 @@ public interface Definition
      */
     boolean isRuntimeSchema();
 
-    /**
-     * Item definition that has this flag set should be ignored by any processing.
-     * The ignored item is still part of the schema. Item instances may appear in
-     * the serialized data formats (e.g. XML) or data store and the parser should
-     * not raise an error if it encounters them. But any high-level processing code
-     * should ignore presence of this item. E.g. it should not be displayed to the user,
-     * should not be present in transformed data structures, etc.
-     *
-     * Note that the same item can be ignored at higher layer (e.g. presentation)
-     * but not ignored at lower layer (e.g. model). This works by presenting different
-     * item definitions for these layers (see LayerRefinedAttributeDefinition).
-     *
-     * Semantics of this flag for complex type definitions is to be defined yet.
-     */
-    // TODO Remove in 4.3, check whether all usages are equivalent to getProcessing() == IGNORE
-    @Deprecated
+    /** Just a shortcut for now. */
     default boolean isIgnored() {
         return getProcessing() == ItemProcessing.IGNORE;
     }
@@ -249,4 +234,10 @@ default String debugDump(int indent, IdentityHashMap<Definition,Object> seen) {
     default String getMutabilityFlag() {
         return isImmutable() ? "" : "+";
     }
+
+    default void checkMutableOnExposing() {
+        if (isImmutable()) {
+            throw new IllegalStateException("Definition couldn't be exposed as mutable because it is immutable: " + this);
+        }
+    }
 }

infra/prism-api/src/main/java/com/evolveum/midpoint/prism/Item.java

@@ -18,6 +18,7 @@
 
 import javax.xml.namespace.QName;
 
+import com.evolveum.midpoint.prism.polystring.PolyString;
 import com.evolveum.midpoint.util.MiscUtil;
 
 import org.apache.commons.lang3.BooleanUtils;
@@ -103,11 +104,8 @@ default boolean hasCompleteDefinition() {
      * Sets applicable item definition.
      *
      * @param definition the definition to set
-     *
-     * TODO consider removing this method
      */
-    @VisibleForTesting
-    void setDefinition(@Nullable D definition);
+    void setDefinition(@NotNull D definition);
 
     /**
      * Returns a display name for the item.
@@ -233,6 +231,7 @@ default V getAnyValue() {
     /**
      * Returns the value, if there is only one. Throws exception if there are more values.
      * If there is no value, this method either:
+     *
      * - returns null (for properties)
      * - throws an exception (for items that can hold multiple values)
      * - creates an empty value (for containers and references).
@@ -280,11 +279,27 @@ default V getAnyValue(@NotNull ValueSelector<V> selector) {
 
     /**
      * Returns (potentially empty) collection of "real values".
-     * @see Item#getRealValue().
+     *
+     * The list itself is detached, freely modifiable.
+     * (Note that the values can still embed a parent, e.g., for containers and references.)
+     *
+     * @see Item#getRealValue()
      */
     @NotNull
     Collection<?> getRealValues();
 
+    /**
+     * Returns detached collection of real values, although the values are still _connected_ to the original item
+     * (in case of complex properties, references, and containers).
+     *
+     * BEWARE, it's not always possible to get the real values.
+     */
+    default <X> @NotNull Collection<X> getRealValues(Class<X> type) {
+        return getRealValues().stream() // TODO should we avoid using streams here?
+                .map(type::cast)
+                .collect(Collectors.toList());
+    }
+
     @Experimental // Do NOT use !!!!
     @NotNull
     default Collection<Object> getRealValuesOrRawTypes(PrismContext prismContext) {
@@ -597,11 +612,23 @@ default void filterYields(BiFunction<V, PrismContainerValue, Boolean> function)
         }
     }
 
-    void applyDefinition(D definition) throws SchemaException;
+    default void applyDefinition(@NotNull D definition) throws SchemaException {
+        applyDefinition(definition, true);
+    }
+
+    default void applyDefinitionIfMissing(@NotNull D definition) throws SchemaException {
+        applyDefinition(definition, false);
+    }
 
-    void applyDefinition(D definition, boolean force) throws SchemaException;
+    /**
+     * Applies the definition to this item (and all its values, down to the lowest level).
+     *
+     * It may even convert the values, e.g. from their raw (unparsed) form to the parsed one,
+     * or - for resource attributes - between {@link String} and {@link PolyString} values (due to the normalization).
+     */
+    void applyDefinition(@NotNull D definition, boolean force) throws SchemaException;
 
-    default Item<?,?> copy() {
+    default Item<V, D> copy() {
         return clone();
     }
 

infra/prism-api/src/main/java/com/evolveum/midpoint/prism/ItemDefinition.java

@@ -9,6 +9,8 @@
 import java.util.Optional;
 import javax.xml.namespace.QName;
 
+import com.evolveum.prism.xml.ns._public.types_3.RawType;
+
 import org.jetbrains.annotations.NotNull;
 
 import com.evolveum.midpoint.prism.delta.ItemDelta;
@@ -23,7 +25,7 @@
  *
  * @see TypeDefinition
  */
-public interface ItemDefinition<I extends Item>
+public interface ItemDefinition<I extends Item<?, ?>>
         extends Definition, PrismItemAccessDefinition {
 
     /**
@@ -185,8 +187,10 @@ default boolean isOptional() {
 
     /**
      * TODO document
+     *
+     * The prism value must not be of {@link RawType}.
      */
-    boolean canBeDefinitionOf(PrismValue pvalue);
+    boolean canBeDefinitionOf(@NotNull PrismValue pvalue);
 
     @Override
     MutableItemDefinition<I> toMutable();
@@ -203,7 +207,6 @@ default boolean isOptional() {
 
     /**
      * Returns true if item definition is searchable.
-     * @return
      */
     @Experimental
     default boolean isSearchable() {

infra/prism-api/src/main/java/com/evolveum/midpoint/prism/MutableComplexTypeDefinition.java

@@ -47,6 +47,8 @@ public interface MutableComplexTypeDefinition extends ComplexTypeDefinition, Mut
 
     void setReferenceMarker(boolean value);
 
+    void setDefaultItemTypeName(QName value);
+
     void setDefaultNamespace(String namespace);
 
     void setIgnoredNamespaces(@NotNull List<String> ignoredNamespaces);

infra/prism-api/src/main/java/com/evolveum/midpoint/prism/MutableDefinition.java

@@ -47,4 +47,91 @@ public interface MutableDefinition extends Definition {
     void addSchemaMigration(SchemaMigration schemaMigration);
 
     void addDiagram(ItemDiagramSpecification diagram);
+
+    /**
+     * A variant of {@link MutableDefinition} that does not allow any modifications. Useful for implementations that want
+     * to allow only selected mutating operations.
+     */
+    interface Unsupported extends MutableDefinition {
+
+        @Override
+        default void setProcessing(ItemProcessing processing) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setDeprecated(boolean deprecated) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setRemoved(boolean removed) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setRemovedSince(String removedSince) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setOptionalCleanup(boolean optionalCleanup) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setExperimental(boolean experimental) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setEmphasized(boolean emphasized) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setDisplayHint(DisplayHint display) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setDisplayName(String displayName) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setDisplayOrder(Integer displayOrder) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setHelp(String help) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setRuntimeSchema(boolean value) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setTypeName(QName typeName) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void setDocumentation(String value) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void addSchemaMigration(SchemaMigration schemaMigration) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        default void addDiagram(ItemDiagramSpecification diagram) {
+            throw new UnsupportedOperationException();
+        }
+    }
 }