Skip to content

Commit

Permalink
Add repo support for index-only ext. values
Browse files Browse the repository at this point in the history 
This is related to MID-5558 (storage of objects with large number of
extension/attribute values). Main goals:

1) avoid storing values of index-only extension items in object XML
2) eliminate "get all" SELECTs from tables for extension values

It is a work in progress.

Related changes:
- refactored ObjectDeltaUpdater a bit;
- added comprehensive test for extension values;
- done a couple of loosely related fixes in delta processing;
- counts for extension items were removed form RObject and
RAssignmentExtension (DB scripts not updated yet.);
- refactored SelectorOptions.hasToLoadPath and related methods.

TODO:
- SELECTs are not really optimized yet. (Only bulk selects are gone.)
- Support for index-only attributes is not there yet.
- RAnyConverter.get[All]ExtValues is not complete.
  • Loading branch information
@mederly
mederly committed Jul 24, 2019
1 parent e2c5899 commit e0939ab
Show file tree
Hide file tree
Showing 61 changed files with 3,977 additions and 2,030 deletions.
2 changes: 2 additions & 0 deletions infra/prism-api/src/main/java/com/evolveum/midpoint/prism/path/ItemPathSegmentUtil.java
View file
Expand Up @@ -16,6 +16,7 @@

package com.evolveum.midpoint.prism.path;

import org.jetbrains.annotations.Contract;
import org.jetbrains.annotations.NotNull;

import javax.xml.namespace.QName;
Expand All @@ -29,6 +30,7 @@ public static boolean isName(Object segment) {
&& !isSpecialName(segment); // todo remove
}

@Contract("_, true -> !null")
static ItemName toName(Object segment, boolean failOnError) {
if (segment instanceof NameItemPathSegment) {
return ((NameItemPathSegment) segment).getName();
Expand Down
1 change: 1 addition & 0 deletions infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/ItemDefinitionImpl.java
View file
Expand Up @@ -364,6 +364,7 @@ protected void copyDefinitionData(ItemDefinitionImpl<I> clone) {
clone.canModify = this.canModify;
clone.operational = this.operational;
clone.valueEnumerationRef = this.valueEnumerationRef;
clone.indexOnly = this.indexOnly;
}

/**
Expand Down
2 changes: 1 addition & 1 deletion infra/schema/src/main/java/com/evolveum/midpoint/schema/RetrieveOption.java
View file
Expand Up @@ -5,7 +5,7 @@
public enum RetrieveOption {

/**
* Return the item "as ususal". If the item would be returned by default then return it.
* Return the item "as usual". If the item would be returned by default then return it.
* If the item would not be returned by default then it may not be returned.
*/
DEFAULT,
Expand Down
124 changes: 32 additions & 92 deletions infra/schema/src/main/java/com/evolveum/midpoint/schema/SelectorOptions.java
View file
Expand Up @@ -20,18 +20,20 @@
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;

import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.path.*;
import com.evolveum.midpoint.util.DebugDumpable;
import com.evolveum.midpoint.util.MiscUtil;
import com.evolveum.midpoint.util.ShortDumpable;
import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import javax.xml.namespace.QName;
import static com.evolveum.midpoint.util.MiscUtil.emptyIfNull;

/**
* @author semancik
Expand Down Expand Up @@ -188,109 +190,47 @@ public boolean isRoot() {

private static final Set<Class<?>> OBJECTS_NOT_RETURNED_FULLY_BY_DEFAULT = new HashSet<>(Arrays.asList(
UserType.class, FocusType.class, AssignmentHolderType.class, ObjectType.class,
TaskType.class, LookupTableType.class, AccessCertificationCampaignType.class
TaskType.class, LookupTableType.class, AccessCertificationCampaignType.class,
ShadowType.class // because of index-only attributes
));

public static boolean isRetrievedFullyByDefault(Class<?> objectType) {
return !OBJECTS_NOT_RETURNED_FULLY_BY_DEFAULT.contains(objectType);
}

public static boolean hasToLoadPath(ItemPath path, Collection<SelectorOptions<GetOperationOptions>> options) {
List<SelectorOptions<GetOperationOptions>> retrieveOptions = filterRetrieveOptions(options);
if (retrieveOptions.isEmpty()) {
return !ItemPathCollectionsUtil.containsEquivalent(PATHS_NOT_RETURNED_BY_DEFAULT, path);
}

for (SelectorOptions<GetOperationOptions> option : retrieveOptions) {
ObjectSelector selector = option.getSelector();
if (selector != null) {
UniformItemPath selected = selector.getPath();
if (!isPathInSelected(path, selected)) {
continue;
}
}

RetrieveOption retrieveOption = option.getOptions().getRetrieve();
for (ItemPath notByDefault : PATHS_NOT_RETURNED_BY_DEFAULT) {
if (path.equivalent(notByDefault)) {
//this one is not retrieved by default
switch (retrieveOption) {
case INCLUDE:
return true;
case EXCLUDE:
case DEFAULT:
default:
return false;
}
}
}

switch (retrieveOption) {
case EXCLUDE:
case DEFAULT:
return false;
case INCLUDE:
default:
return true;
}
}

return false;
}

public static boolean isExplicitlyIncluded(UniformItemPath path, Collection<SelectorOptions<GetOperationOptions>> options) {
List<SelectorOptions<GetOperationOptions>> retrieveOptions = filterRetrieveOptions(options);
if (retrieveOptions.isEmpty()) {
return false;
}

for (SelectorOptions<GetOperationOptions> option : retrieveOptions) {
ObjectSelector selector = option.getSelector();
if (selector != null) {
UniformItemPath selected = selector.getPath();
if (!isPathInSelected(path, selected)) {
continue;
}
}

RetrieveOption retrieveOption = option.getOptions().getRetrieve();
switch (retrieveOption) {
case INCLUDE:
return true;
case EXCLUDE:
case DEFAULT:
default:
return false;
}
}

return false;
}
public static boolean hasToLoadPath(@NotNull ItemPath path, Collection<SelectorOptions<GetOperationOptions>> options) {
return hasToLoadPath(path, options, !ItemPathCollectionsUtil.containsEquivalent(PATHS_NOT_RETURNED_BY_DEFAULT, path));
}

private static boolean isPathInSelected(ItemPath path, ItemPath selected) {
if (selected == null || path == null) {
return false;
} else {
return selected.isSubPathOrEquivalent(path);
public static boolean hasToLoadPath(@NotNull ItemPath path, Collection<SelectorOptions<GetOperationOptions>> options,
boolean defaultValue) {
for (SelectorOptions<GetOperationOptions> option : emptyIfNull(options)) {
// TODO consider ordering of the options from most specific to least specific
RetrieveOption retrievalCommand = option != null && option.getOptions() != null ? option.getOptions().getRetrieve() : null;
if (retrievalCommand != null) {
ObjectSelector selector = option.getSelector();
if (selector == null || selector.getPath() == null || selector.getPath().isSubPathOrEquivalent(path)) {
switch (retrievalCommand) {
case EXCLUDE:
return false;
case DEFAULT:
return defaultValue;
case INCLUDE:
return true;
default:
throw new AssertionError("Wrong retrieve option: " + retrievalCommand);
}
}
}
}
return defaultValue;
}

public static List<SelectorOptions<GetOperationOptions>> filterRetrieveOptions(
Collection<SelectorOptions<GetOperationOptions>> options) {
List<SelectorOptions<GetOperationOptions>> retrieveOptions = new ArrayList<>();
if (options == null) {
return retrieveOptions;
}

for (SelectorOptions<GetOperationOptions> option : options) {
if (option.getOptions() == null || option.getOptions().getRetrieve() == null) {
continue;
}

retrieveOptions.add(option);
}

return retrieveOptions;
return MiscUtil.streamOf(options)
.filter(option -> option.getOptions() != null && option.getOptions().getRetrieve() != null)
.collect(Collectors.toList());
}

public static <T> Map<T, Collection<UniformItemPath>> extractOptionValues(Collection<SelectorOptions<GetOperationOptions>> options,
Expand Down
10 changes: 10 additions & 0 deletions infra/util/src/main/java/com/evolveum/midpoint/util/MiscUtil.java
View file
Expand Up @@ -642,6 +642,16 @@ public static <T> List<T> emptyIfNull(List<T> list) {
return list != null ? list : Collections.emptyList();
}

@NotNull
public static <T> Collection<T> emptyIfNull(Collection<T> collection) {
return collection != null ? collection : Collections.emptyList();
}

@NotNull
public static <T> Stream<T> streamOf(Collection<T> collection) {
return collection != null ? collection.stream() : Stream.empty();
}

public static String nullIfEmpty(String s) {
return "".equals(s) ? null : s;
}
Expand Down
2 changes: 1 addition & 1 deletion ...isioning-impl/src/main/java/com/evolveum/midpoint/provisioning/util/ProvisioningUtil.java
View file
Expand Up @@ -168,7 +168,7 @@ public static AttributesToReturn createAttributesToReturn(ProvisioningContext ct
if (fetchStrategy != null && fetchStrategy == AttributeFetchStrategyType.EXPLICIT) {
explicit.add(attributeDefinition);
} else if (hasMinimal && (fetchStrategy != AttributeFetchStrategyType.MINIMAL ||
SelectorOptions.isExplicitlyIncluded(ctx.getPrismContext().toUniformPath(attributeDefinition.getName()), ctx.getGetOperationOptions()))) {
SelectorOptions.hasToLoadPath(ctx.getPrismContext().toUniformPath(attributeDefinition.getName()), ctx.getGetOperationOptions(), false))) {
explicit.add(attributeDefinition);
}
}
Expand Down
11 changes: 7 additions & 4 deletions repo/repo-api/src/main/java/com/evolveum/midpoint/repo/api/RepoModifyOptions.java
View file
Expand Up @@ -21,14 +21,15 @@
import com.evolveum.midpoint.util.ShortDumpable;

/**
* @author mederly
*
*/
public class RepoModifyOptions extends AbstractOptions implements Serializable, ShortDumpable {
private static final long serialVersionUID = 478427843213482L;

// execute MODIFY operation even if the list of changes is empty
private boolean executeIfNoChanges = false;
/**
* Execute MODIFY operation even if the list of changes is empty.
*/
private boolean executeIfNoChanges;

public boolean isExecuteIfNoChanges() {
return executeIfNoChanges;
Expand All @@ -39,14 +40,16 @@ public void setExecuteIfNoChanges(boolean executeIfNoChanges) {
}

public static boolean isExecuteIfNoChanges(RepoModifyOptions options) {
return options != null ? options.isExecuteIfNoChanges() : false;
return options != null && options.isExecuteIfNoChanges();
}

public static RepoModifyOptions createExecuteIfNoChanges() {
RepoModifyOptions opts = new RepoModifyOptions();
opts.setExecuteIfNoChanges(true);
return opts;
}



@Override
public String toString() {
Expand Down
38 changes: 27 additions & 11 deletions ...po/sql/testing/QueryCountInterceptor.java → ...oint/repo/sql/testing/QueryInspector.java
View file
Expand Up @@ -18,33 +18,49 @@

import org.hibernate.EmptyInterceptor;

import java.util.ArrayList;
import java.util.List;

/**
* Created by Viliam Repan (lazyman).
*/
public class QueryCountInterceptor extends EmptyInterceptor {
public class QueryInspector extends EmptyInterceptor {

private ThreadLocal<Integer> queryCount = new ThreadLocal<>();
private ThreadLocal<List<String>> queryCount = new ThreadLocal<>();

public void startCounter() {
queryCount.set(0);
public void start() {
queryCount.set(new ArrayList<>());
}

public int getQueryCount() {
Integer i = queryCount.get();
return i == null ? 0 : i;
List<String> queries = getQueries();
return queries != null ? queries.size() : 0;
}

public List<String> getQueries() {
return queryCount.get();
}

public void clearCounter() {
public void clear() {
queryCount.remove();
}

@Override
public String onPrepareStatement(String sql) {
Integer count = queryCount.get();
if (count != null) {
queryCount.set(count + 1);
List<String> queries = getQueries();
if (queries != null) {
queries.add(sql);
}

return super.onPrepareStatement(sql);
}

public void dump() {
List<String> queries = getQueries();
if (queries != null) {
System.out.println("Queries collected (" + queries.size() + "):");
queries.forEach(q -> System.out.println(" - " + q));
} else {
System.out.println("Query collection was not started for this thread.");
}
}
}
4 changes: 2 additions & 2 deletions ...o-sql-impl-test/src/main/java/com/evolveum/midpoint/repo/sql/testing/TestInterceptor.java
View file
Expand Up @@ -29,7 +29,7 @@ public class TestInterceptor extends EmptyInterceptor {
private EntityStateInterceptor entityStateInterceptor;

@Autowired
private QueryCountInterceptor queryCountInterceptor;
private QueryInspector queryInspector;

@Override
public Boolean isTransient(Object entity) {
Expand All @@ -38,6 +38,6 @@ public Boolean isTransient(Object entity) {

@Override
public String onPrepareStatement(String sql) {
return queryCountInterceptor.onPrepareStatement(sql);
return queryInspector.onPrepareStatement(sql);
}
}
8 changes: 3 additions & 5 deletions .../main/java/com/evolveum/midpoint/repo/sql/testing/TestSqlRepositoryBeanPostProcessor.java
View file
Expand Up @@ -41,10 +41,8 @@ public class TestSqlRepositoryBeanPostProcessor implements BeanPostProcessor {
private static final String TRUNCATE_FUNCTION = "cleanupTestDatabase";
private static final String TRUNCATE_PROCEDURE = "cleanupTestDatabaseProc";

@Autowired
private ApplicationContext context;
@Autowired
private QueryCountInterceptor queryCountInterceptor;
@Autowired private ApplicationContext context;
@Autowired private QueryInspector queryInspector;

@Override
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
Expand All @@ -69,7 +67,7 @@ public Object postProcessAfterInitialization(Object bean, String beanName) throw
LOGGER.info("Deleting objects from database.");

SessionFactory sessionFactory = (SessionFactory) bean;
sessionFactory.withOptions().interceptor(queryCountInterceptor);
sessionFactory.withOptions().interceptor(queryInspector);
Session session = sessionFactory.openSession();
try {
session.beginTransaction();
Expand Down
2 changes: 1 addition & 1 deletion repo/repo-sql-impl-test/src/main/resources/ctx-repository-test.xml
View file
Expand Up @@ -21,7 +21,7 @@
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd"
default-lazy-init="true" default-autowire="byName">

<bean id="queryCountInterceptor" class="com.evolveum.midpoint.repo.sql.testing.QueryCountInterceptor"/>
<bean id="queryInspector" class="com.evolveum.midpoint.repo.sql.testing.QueryInspector"/>

<bean id="entityStateInterceptor" class="com.evolveum.midpoint.repo.sql.util.EntityStateInterceptor"/>

Expand Down

0 comments on commit e0939ab

Please sign in to comment.