Merge remote-tracking branch 'origin/master'

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/ExtensionProcessor.java

@@ -53,8 +53,8 @@ public Jsonb processExtensions(
 
                 extMap.put(extItemInfo.getId(), value);
 
-                // we may need to add add also single value index, if definition is dynamic
-                // see additionalSingleValueIndexNeeded javadoc for more information
+                // We may need to add also single value index, if definition is dynamic;
+                // see additionalSingleValueIndexNeeded() javadoc for more information.
                 if (additionalSingleValueIndexNeeded(item, extItemInfo)) {
                     addSingleValueIndex(extMap, item, holderType);
                 }

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepositoryService.java

@@ -18,8 +18,6 @@
 import java.util.stream.Collectors;
 import javax.xml.namespace.QName;
 
-import com.evolveum.midpoint.prism.util.PrismUtil;
-
 import com.google.common.base.Strings;
 import com.google.common.collect.ObjectArrays;
 import com.querydsl.core.Tuple;
@@ -43,6 +41,7 @@
 import com.evolveum.midpoint.prism.query.*;
 import com.evolveum.midpoint.prism.query.builder.S_ConditionEntry;
 import com.evolveum.midpoint.prism.query.builder.S_MatchingRuleEntry;
+import com.evolveum.midpoint.prism.util.PrismUtil;
 import com.evolveum.midpoint.prism.xml.XmlTypeConverter;
 import com.evolveum.midpoint.repo.api.*;
 import com.evolveum.midpoint.repo.api.query.ObjectFilterExpressionEvaluator;
@@ -109,6 +108,12 @@ public class SqaleRepositoryService extends SqaleServiceBase implements Reposito
     private static final Collection<SelectorOptions<GetOperationOptions>> GET_FOR_UPDATE_OPTIONS =
             SchemaService.get().getOperationOptionsBuilder().build();
 
+    private static final Collection<SelectorOptions<GetOperationOptions>> GET_FOR_REINDEX_OPTIONS =
+            SchemaService.get().getOperationOptionsBuilder()
+                    .retrieve()
+                    .raw()
+                    .build();
+
     private final SqlQueryExecutor sqlQueryExecutor;
 
     @Autowired private SystemConfigurationChangeDispatcher systemConfigurationChangeDispatcher;
@@ -656,13 +661,16 @@ RootUpdateContext<S, Q, R> prepareUpdateContext(
             @NotNull JdbcSession jdbcSession,
             @NotNull Class<S> schemaType,
             @NotNull Collection<? extends ItemDelta<?, ?>> modifications,
-            @NotNull UUID oid, RepoModifyOptions options)
+            @NotNull UUID oid,
+            @Nullable RepoModifyOptions options)
             throws SchemaException, ObjectNotFoundException {
 
         QueryTableMapping<S, FlexibleRelationalPathBase<Object>, Object> rootMapping =
                 sqlRepoContext.getMappingBySchemaType(schemaType);
         Collection<SelectorOptions<GetOperationOptions>> getOptions =
-                rootMapping.updateGetOptions(GET_FOR_UPDATE_OPTIONS, modifications);
+                rootMapping.updateGetOptions(
+                        RepoModifyOptions.isForceReindex(options) ? GET_FOR_REINDEX_OPTIONS : GET_FOR_UPDATE_OPTIONS,
+                        modifications);
 
         return prepareUpdateContext(jdbcSession, schemaType, oid, getOptions, options);
     }

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/object/QObjectMapping.java

@@ -131,6 +131,9 @@ protected QObjectMapping(
     @Override
     public @NotNull Path<?>[] selectExpressions(
             Q entity, Collection<SelectorOptions<GetOperationOptions>> options) {
+        // TODO: there is currently no support for index-only extensions (from entity.ext).
+        //  See how QShadowMapping.loadIndexOnly() is used, and probably compose the result of this call
+        //  using super... call in the subclasses. (joining arrays? providing mutable list?)
         return new Path[] { entity.oid, entity.fullObject };
     }
 

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/SqaleRepoBaseTest.java

@@ -574,6 +574,10 @@ public class ShadowAttributesHelper {
         private final ShadowAttributesType attributesContainer;
         private final MutablePrismContainerDefinition<Containerable> attrsDefinition;
 
+        /**
+         * Creates the attribute helper for the shadow, adding attributes container to the shadow.
+         * The container can be later obtained by {@link #attributesContainer()} if/when needed.
+         */
         public ShadowAttributesHelper(ShadowType object) throws SchemaException {
             attributesContainer = new ShadowAttributesType();
             // let's create the container+PCV inside the shadow object

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoModifyObjectTest.java

@@ -64,6 +64,7 @@
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.schema.util.ObjectTypeUtil;
 import com.evolveum.midpoint.test.util.TestUtil;
+import com.evolveum.midpoint.util.DOMUtil;
 import com.evolveum.midpoint.util.MiscUtil;
 import com.evolveum.midpoint.util.exception.ObjectAlreadyExistsException;
 import com.evolveum.midpoint.util.exception.ObjectNotFoundException;
@@ -3534,8 +3535,8 @@ public void test950ModifyOperationWithReindexUpdatesPerformanceMonitor()
         OperationResult result = createOperationResult();
         given("Full object was modified in database");
         clearPerformanceMonitor();
-        when("object is modified in the repository");
 
+        when("object is modified in the repository");
         RepoModifyOptions options = RepoModifyOptions.createForceReindex();
         repositoryService.modifyObject(UserType.class, user1Oid, Collections.emptyList(), options, result);
 
@@ -3549,19 +3550,17 @@ public void test951ReindexAfterManualChangeOfFullObject()
             throws SchemaException, ObjectNotFoundException, ObjectAlreadyExistsException {
         OperationResult result = createOperationResult();
 
+        given("user existing in the repository");
         UserType user = new UserType().name("corrupted")
                 .beginAssignment()
                 .policySituation("kept")
                 .<UserType>end()
                 .beginAssignment()
                 .policySituation("removed")
                 .end();
-
-        String oid = repositoryService.addObject(user.asPrismObject(),
-                null, result);
+        String oid = repositoryService.addObject(user.asPrismObject(), null, result);
 
         when("full object is modified in the database (indices are desynced from full object)");
-
         user = repositoryService.getObject(UserType.class, oid, null, result).asObjectable();
         user.getAssignment().remove(1); // remove removed
         user.beginAssignment().policySituation("added");
@@ -3579,8 +3578,7 @@ public void test951ReindexAfterManualChangeOfFullObject()
         assertPolicySituationFound("removed", 1, result);
         assertPolicySituationFound("added", 0, result);
 
-        given("object is reindexed");
-
+        and("object is reindexed");
         RepoModifyOptions options = RepoModifyOptions.createForceReindex();
         repositoryService.modifyObject(UserType.class, oid, Collections.emptyList(), options, result);
 
@@ -3692,6 +3690,40 @@ public void test952ReindexFixingColumnsOutOfSync()
         assertThat(assRows.get(0).orderValue).isEqualTo(1);
     }
 
+    @Test
+    public void test955ReindexOfShadowWithAttributes() throws Exception {
+        OperationResult result = createOperationResult();
+
+        given("delta adding an attributes container for for shadow 1");
+        ShadowType shadow = new ShadowType()
+                .name("shadow-" + getTestName());
+        ShadowAttributesType attributesContainer = new ShadowAttributesHelper(shadow)
+                .set(new QName("https://example.com/p", "string-mv"), DOMUtil.XSD_STRING,
+                        "string-value1", "string-value2")
+                .attributesContainer();
+        String oid = repositoryService.addObject(shadow.asPrismObject(), null, result);
+
+        when("reindex is called");
+        repositoryService.modifyObject(
+                ShadowType.class, oid, List.of(), RepoModifyOptions.createForceReindex(), result);
+
+        then("operation is successful");
+        assertThatOperationResult(result).isSuccess();
+
+        and("serialized form (fullObject) still has attributes");
+        ShadowType shadowObject = repositoryService.getObject(ShadowType.class, oid, null, result)
+                .asObjectable();
+        assertThat(shadowObject.getAttributes()).isNotNull();
+
+        and("externalized attributes are also preserved");
+        MShadow row = selectObjectByOid(QShadow.class, oid);
+        assertThat(row.attributes).isNotNull();
+        Map<String, Object> attributeMap = Jsonb.toMap(row.attributes);
+        assertThat(attributeMap).containsEntry(
+                shadowAttributeKey(attributesContainer, "string-mv"),
+                List.of("string-value1", "string-value2"));
+    }
+
     @Test
     public void test990ChangeOfNonPersistedItemWorksOk()
             throws ObjectAlreadyExistsException, ObjectNotFoundException, SchemaException {

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoSmokeTest.java

@@ -12,10 +12,7 @@
 import static com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase.DEFAULT_SCHEMA_NAME;
 
 import java.sql.SQLException;
-import java.util.Collection;
-import java.util.Date;
-import java.util.Queue;
-import java.util.UUID;
+import java.util.*;
 
 import org.testng.Assert;
 import org.testng.SkipException;
@@ -28,6 +25,7 @@
 import com.evolveum.midpoint.prism.PrismObject;
 import com.evolveum.midpoint.prism.xml.XmlTypeConverter;
 import com.evolveum.midpoint.repo.api.DeleteObjectResult;
+import com.evolveum.midpoint.repo.api.RepoModifyOptions;
 import com.evolveum.midpoint.repo.api.RepositoryService;
 import com.evolveum.midpoint.repo.sqale.SqaleRepoBaseTest;
 import com.evolveum.midpoint.repo.sqale.jsonb.Jsonb;
@@ -62,6 +60,8 @@
  */
 public class SqaleRepoSmokeTest extends SqaleRepoBaseTest {
 
+    public static final byte[] JPEG_PHOTO = { 0, 1, 2 }; // not really a JPEG, of course
+
     @AfterMethod
     public void methodCleanup() {
         queryRecorder.stopRecording();
@@ -319,13 +319,13 @@ public void test220PhotoPersistenceAdd()
         when("user with photo is persisted");
         UserType user = new UserType()
                 .name("user" + getTestNumber())
-                .jpegPhoto(new byte[] { 0, 1, 2 });
+                .jpegPhoto(JPEG_PHOTO);
         String userOid = repositoryService.addObject(user.asPrismObject(), null, result);
         assertThatOperationResult(result).isSuccess();
 
         then("photo is stored in row, but not in fullObject");
         MUser row = selectObjectByOid(QUser.class, UUID.fromString(userOid));
-        assertThat(row.photo).isEqualTo(new byte[] { 0, 1, 2 });
+        assertThat(row.photo).isEqualTo(JPEG_PHOTO);
         UserType fullObjectUser = parseFullObject(row.fullObject);
         assertThat(fullObjectUser.getJpegPhoto()).isNull();
 
@@ -341,7 +341,7 @@ public void test220PhotoPersistenceAdd()
         UserType userWithPhoto =
                 repositoryService.getObject(UserType.class, userOid, photoOptions, result)
                         .asObjectable();
-        assertThat(userWithPhoto.getJpegPhoto()).isEqualTo(new byte[] { 0, 1, 2 });
+        assertThat(userWithPhoto.getJpegPhoto()).isEqualTo(JPEG_PHOTO);
         assertThat(userWithPhoto.asPrismObject().findProperty(FocusType.F_JPEG_PHOTO).isIncomplete()).isFalse();
     }
 
@@ -359,14 +359,37 @@ public void test221PhotoPersistenceModify()
         //noinspection PrimitiveArrayArgumentToVarargsMethod
         repositoryService.modifyObject(UserType.class, userOid,
                 prismContext.deltaFor(UserType.class)
-                        .item(UserType.F_JPEG_PHOTO).add(new byte[] { 0, 1, 2 })
+                        .item(UserType.F_JPEG_PHOTO).add(JPEG_PHOTO)
                         .asObjectDelta(userOid).getModifications(),
                 result);
         assertThatOperationResult(result).isSuccess();
 
         then("photo is stored in row, but not in fullObject");
         MUser row = selectObjectByOid(QUser.class, UUID.fromString(userOid));
-        assertThat(row.photo).isEqualTo(new byte[] { 0, 1, 2 });
+        assertThat(row.photo).isEqualTo(JPEG_PHOTO);
+        UserType fullObjectUser = parseFullObject(row.fullObject);
+        assertThat(fullObjectUser.getJpegPhoto()).isNull();
+    }
+
+    @Test
+    public void test222PhotoPersistenceReindex()
+            throws SchemaException, ObjectAlreadyExistsException, ObjectNotFoundException {
+        OperationResult result = createOperationResult();
+
+        given("user with photo");
+        UserType user = new UserType()
+                .name("user" + getTestNumber())
+                .jpegPhoto(JPEG_PHOTO);
+        String userOid = repositoryService.addObject(user.asPrismObject(), null, result);
+
+        when("user is reindexed");
+        repositoryService.modifyObject(UserType.class, userOid,
+                List.of(), RepoModifyOptions.createForceReindex(), result);
+        assertThatOperationResult(result).isSuccess();
+
+        then("photo is still in row, but not in fullObject");
+        MUser row = selectObjectByOid(QUser.class, UUID.fromString(userOid));
+        assertThat(row.photo).isEqualTo(JPEG_PHOTO);
         UserType fullObjectUser = parseFullObject(row.fullObject);
         assertThat(fullObjectUser.getJpegPhoto()).isNull();
     }