sqale: Migrate non-splitted object to splitted during modify operation

When object partials are read from repository during update mark object for reindex if any partial is missing full object. This should migrate objects stored before 4.8 to 4.9 format.
Evolveum · Apr 3, 2024 · e45ab75 · e45ab75
1 parent a55bc5f
commit e45ab75
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 10 deletions.
diff --git a/repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepositoryService.java b/repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepositoryService.java
@@ -657,8 +657,10 @@ private <T extends ObjectType> ModifyObjectResult<T> modifyObjectInternal(
             invokeConflictWatchers(w -> w.beforeModifyObject(prismObject));
             PrismObject<T> originalObject = prismObject.clone(); // for result later
 
-            boolean reindex = options.isForceReindex();
 
+            // Use reindex instead of modify if reindex is required by user, or repository
+            // itself detected need for reindex during preparation read for modify.
+            boolean reindex = updateContext.reindexNeeded() || options.isForceReindex();
             if (reindex) {
                 // UpdateTables is false, we want only to process modifications on fullObject
                 // do not modify nested items.

diff --git a/repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleUtils.java b/repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleUtils.java
@@ -49,6 +49,7 @@ public class SqaleUtils {
      */
     public static final String OWNER_OID = "ownerOid";
     public static final String FULL_ID_PATH = "containerIdPath";
+    public static final String REINDEX_NEEDED = "sqale.reindexNeeded";
 
     /**
      * Returns version from midPoint object as a number.

diff --git a/.../src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/assignment/QAssignmentMapping.java b/.../src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/assignment/QAssignmentMapping.java
@@ -42,6 +42,8 @@
 import com.evolveum.midpoint.repo.sqlbase.mapping.TableRelationResolver;
 import com.evolveum.midpoint.util.MiscUtil;
 
+import org.jetbrains.annotations.VisibleForTesting;
+
 /**
  * Mapping between {@link QAssignment} and {@link AssignmentType}.
  * There are separate instances for assignments and inducements and the instance also knows
@@ -62,6 +64,9 @@ public class QAssignmentMapping<OR extends MObject>
     /** Inducement mapping instance, this must be used for inserting inducements. */
     private static QAssignmentMapping<?> instanceInducement;
 
+    private boolean storeFullObject = true;
+
+
     // Explanation in class Javadoc for SqaleTableMapping
     public static <OR extends MObject> QAssignmentMapping<OR>
     initAssignmentMapping(@NotNull SqaleRepoContext repositoryContext) {
@@ -283,7 +288,7 @@ public AssignmentType toSchemaObjectLegacy(MAssignment row) {
     @Override
     public MAssignment insert(AssignmentType assignment, OR ownerRow, JdbcSession jdbcSession) throws SchemaException {
         assignment = assignment.clone(); // initRowObejctWithFullObject normalizes relations, this modifies delta
-        MAssignment row = initRowObjectWithFullObject(assignment, ownerRow);
+        MAssignment row = storeFullObject ? initRowObjectWithFullObject(assignment, ownerRow) : initRowObject(assignment, ownerRow);
 
         row.lifecycleState = assignment.getLifecycleState();
         row.orderValue = assignment.getOrder();
@@ -370,4 +375,10 @@ public Predicate allOwnedBy(QAssignment<OR> orqAssignment, Collection<UUID> oidL
     public OrderSpecifier<?> orderSpecifier(QAssignment<OR> orqAssignment) {
         return new OrderSpecifier<>(Order.ASC, orqAssignment.cid);
     }
+
+
+    @VisibleForTesting
+    public void setStoreFullObject(boolean value) {
+        storeFullObject = value;
+    }
 }
diff --git a/...po-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/object/QObjectMapping.java b/...po-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/object/QObjectMapping.java
@@ -10,7 +10,6 @@
 
 import java.util.*;
 import java.util.function.BiFunction;
-import java.util.stream.Collectors;
 
 import com.evolveum.midpoint.prism.*;
 import com.evolveum.midpoint.prism.path.ItemName;
@@ -29,6 +28,7 @@
 
 import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
 import com.evolveum.midpoint.schema.RetrieveOption;
+import com.evolveum.midpoint.util.Holder;
 import com.evolveum.midpoint.util.exception.SystemException;
 
 import com.google.common.collect.*;
@@ -433,7 +433,7 @@ public boolean isIncluded(Collection<SelectorOptions<GetOperationOptions>> optio
             return SelectorOptions.hasToFetchPathNotRetrievedByDefault(getPath(), options);
         }
 
-        public Multimap<UUID, Tuple> fetchChildren(Collection<UUID> oidList, JdbcSession jdbcSession) throws SchemaException {
+        public Multimap<UUID, Tuple> fetchChildren(Collection<UUID> oidList, JdbcSession jdbcSession, Set<UUID> toMigrate) throws SchemaException {
             Multimap<UUID, Tuple> ret = MultimapBuilder.hashKeys().arrayListValues().build();
 
             var q = mapping.createAlias();
@@ -444,8 +444,12 @@ public Multimap<UUID, Tuple> fetchChildren(Collection<UUID> oidList, JdbcSession
                     .orderBy(mapping.orderSpecifier(q));
             for (var row : query.fetch()) {
                 // All assignments should have full object present / legacy assignments should be kept
+                var owner =  mapping.getOwner(row,q);
                 if (mapping.hasFullObject(row,q)) {
-                    ret.put(mapping.getOwner(row,q), row);
+                    ret.put(owner, row);
+                } else {
+                    // Indexed value did not contained full object, we should mark it for reindex
+                    toMigrate.add(owner);
                 }
             }
             return ret;
@@ -485,7 +489,11 @@ public ResultListRowTransformer<S, Q, R> createRowTransformer(SqlQueryContext<S,
         Map<MObjectType, Set<FullObjectItemMapping>> itemsToFetch = new HashMap<>();
         Multimap<FullObjectItemMapping, UUID> oidsToFetch = HashMultimap.create();
 
+        // Set of objects, which should be reindexed (they are stored without full objects in nested tables)
+        Set<UUID> objectsToReindex = new HashSet<>();
+
         Map<FullObjectItemMapping, Multimap<UUID, PrismValue>> mappingToData = new HashMap<>();
+
         return new ResultListRowTransformer<S, Q, R>() {
 
             @Override
@@ -522,7 +530,7 @@ public void beforeTransformation(List<Tuple> tuples, Q entityPath) throws Schema
                 for (var mapping : mappingToData.entrySet()) {
                     var result = SqlBaseOperationTracker.fetchChildren(mapping.getKey().mapping.tableName());
                     try {
-                        mapping.setValue(mapping.getKey().fetchChildren(oidsToFetch.get(mapping.getKey()), jdbcSession));
+                        mapping.setValue(mapping.getKey().fetchChildren(oidsToFetch.get(mapping.getKey()), jdbcSession, objectsToReindex));
                     } finally {
                         result.close();
                     }
@@ -537,6 +545,12 @@ public S transform(Tuple tuple, Q entityPath) {
                 if (!storeSplitted) {
                     return baseObject;
                 }
+                if (objectsToReindex.contains(uuid)) {
+                    // Object is in legacy form (splitted items does not have full object, reindex is recommended
+                    // This mark is checked during update from original state read by repository
+                    // which forces reindex as part of udpate
+                    baseObject.asPrismObject().setUserData(SqaleUtils.REINDEX_NEEDED, true);
+                }
                 var childrenResult = SqlBaseOperationTracker.parseChildren("all");
                 try {
                     for (var entry : mappingToData.entrySet()) {
@@ -564,6 +578,7 @@ public int additionalSelectsByDefault() {
         return 0;
     }
 
+    @VisibleForTesting
     public void setStoreSplitted(boolean storeSplitted) {
         this.storeSplitted = storeSplitted;
         fullObjectSkips = null; // Needs to be recomputed

diff --git a/repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/update/RootUpdateContext.java b/repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/update/RootUpdateContext.java
@@ -235,4 +235,15 @@ public PrismObject<S> getPrismObject() {
         //noinspection unchecked
         return (PrismObject<S>) object.asPrismObject();
     }
+
+    /**
+     * Returns true if reindex is needed for this object
+     *
+     * During read problems were found in object storage -  such as data which should be stored in separate tables
+     * were stored in full object, full objects for assignments were missing, etc.
+     *
+     */
+    public boolean reindexNeeded() {
+        return getPrismObject().getUserData().get(SqaleUtils.REINDEX_NEEDED) == Boolean.TRUE;
+    }
 }
diff --git a/...ale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoAddDeleteObjectTest.java b/...ale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoAddDeleteObjectTest.java
@@ -23,6 +23,8 @@
 import java.util.UUID;
 import javax.xml.namespace.QName;
 
+import com.evolveum.midpoint.repo.sqale.qmodel.focus.*;
+
 import org.testng.annotations.Test;
 
 import com.evolveum.midpoint.prism.Containerable;
@@ -46,10 +48,6 @@
 import com.evolveum.midpoint.repo.sqale.qmodel.connector.MConnectorHost;
 import com.evolveum.midpoint.repo.sqale.qmodel.connector.QConnector;
 import com.evolveum.midpoint.repo.sqale.qmodel.connector.QConnectorHost;
-import com.evolveum.midpoint.repo.sqale.qmodel.focus.MFocus;
-import com.evolveum.midpoint.repo.sqale.qmodel.focus.MUser;
-import com.evolveum.midpoint.repo.sqale.qmodel.focus.QGenericObject;
-import com.evolveum.midpoint.repo.sqale.qmodel.focus.QUser;
 import com.evolveum.midpoint.repo.sqale.qmodel.lookuptable.MLookupTableRow;
 import com.evolveum.midpoint.repo.sqale.qmodel.lookuptable.QLookupTableRow;
 import com.evolveum.midpoint.repo.sqale.qmodel.node.MNode;
@@ -239,6 +237,44 @@ public void test113AddWithOverwriteOptionDifferentTypes()
         assertThat(count(QObject.CLASS)).isEqualTo(baseCount + 1);
     }
 
+    @Test
+    public void test119ModifyLegacyPreservesData() throws SchemaException, ObjectAlreadyExistsException, ObjectNotFoundException {
+        OperationResult result = createOperationResult();
+        QUserMapping.getUserMapping().setStoreSplitted(false);
+        QAssignmentMapping.getAssignmentMapping().setStoreFullObject(false);
+        long baseCount = count(QObject.CLASS);
+
+        UserType user = new UserType().name("user" + getTestNumber())
+                .assignment(new AssignmentType().targetRef(UUID.randomUUID().toString(), RoleType.COMPLEX_TYPE));
+        try {
+            given("user already in the repository with legacy format (assignment.full_object is null)");
+            String oid = repositoryService.addObject(user.asPrismObject(), null, result);
+            assertThat(count(QObject.CLASS)).isEqualTo(baseCount + 1);
+
+            QUserMapping.getUserMapping().setStoreSplitted(true);
+            QAssignmentMapping.getAssignmentMapping().setStoreFullObject(true);
+            expect("should be readed correctly with assignment present");
+            UserType readed = repositoryService.getObject(UserType.class, oid, null, result).asObjectable();
+            assertThat(readed.getAssignment()).hasSize(1);
+            // Modify with splitted -
+
+            and("when unrelated modification is applied");
+            var deltas = prismContext.deltaFor(UserType.class)
+                            .item(UserType.F_ACTIVATION).add(new ActivationType().administrativeStatus(ActivationStatusType.ENABLED))
+                            .asItemDeltas();
+            repositoryService.modifyObject(UserType.class, oid, deltas, result);
+            and("assignment should be still there and reindexed");
+            readed = repositoryService.getObject(UserType.class, oid, null, result).asObjectable();
+            assertThat(readed.getAssignment()).hasSize(1);
+            assertThat(readed.getActivation()).isNotNull();
+
+
+        } finally {
+            QUserMapping.getUserMapping().setStoreSplitted(true);
+            QAssignmentMapping.getAssignmentMapping().setStoreFullObject(true);
+        }
+    }
+
     // detailed container tests are from test200 on, this one has overwrite priority :-)
     @Test
     public void test115OverwriteWithContainers()