Skip to content

Commit

Permalink
Add textInfoColumnSize SQL configuration property
Browse files Browse the repository at this point in the history 
This is to support full-text search for MySQL utf8mb4 encoding that
limits m_object_text_info.text size to 191 characters. (Users should
set this value explicitly in the config file.)
  • Loading branch information
@mederly
mederly committed Sep 4, 2019
1 parent 50868e1 commit a64e9d7
Show file tree
Hide file tree
Showing 10 changed files with 48 additions and 20 deletions.
2 changes: 2 additions & 0 deletions ...l-test/src/main/java/com/evolveum/midpoint/repo/sql/testing/TestSqlRepositoryFactory.java
View file
Expand Up @@ -118,6 +118,8 @@ private void updateConfigurationFromProperties(Configuration configuration, Prop
updateConfigurationIntegerProperty(configuration, properties, PROPERTY_MIN_POOL_SIZE);
updateConfigurationIntegerProperty(configuration, properties, PROPERTY_MAX_POOL_SIZE);

updateConfigurationIntegerProperty(configuration, properties, PROPERTY_TEXT_INFO_COLUMN_SIZE);

// Dirty hack, in order to make DataSourceTest happy: if none of database, driver, dialect, embedded is
// present but data source is, let us assume we use H2.
//
Expand Down
10 changes: 10 additions & 0 deletions ...epo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryConfiguration.java
View file
Expand Up @@ -321,6 +321,8 @@ public static IncompatibleSchemaAction fromValue(String text) {
public static final String PROPERTY_ENABLE_NO_FETCH_EXTENSION_VALUES_DELETION = "enableNoFetchExtensionValuesDeletion";
public static final String PROPERTY_ENABLE_INDEX_ONLY_ITEMS = "enableIndexOnlyItems";

public static final String PROPERTY_TEXT_INFO_COLUMN_SIZE = "textInfoColumnSize";

private static final String DRIVER_H2 = Driver.class.getName();
private static final String DRIVER_MYSQL = "com.mysql.cj.jdbc.Driver";
private static final String DRIVER_MARIADB = "org.mariadb.jdbc.Driver";
Expand Down Expand Up @@ -400,6 +402,8 @@ public static IncompatibleSchemaAction fromValue(String text) {
private boolean enableNoFetchExtensionValuesDeletion;
private boolean enableIndexOnlyItems;

private int textInfoColumnSize;

/*
* Notes:
* - In testing mode, the configuration is already updated from .properties file.
Expand Down Expand Up @@ -521,6 +525,8 @@ public SqlRepositoryConfiguration(Configuration configuration) {
enableNoFetchExtensionValuesInsertion = configuration.getBoolean(PROPERTY_ENABLE_NO_FETCH_EXTENSION_VALUES_INSERTION, true);
enableNoFetchExtensionValuesDeletion = configuration.getBoolean(PROPERTY_ENABLE_NO_FETCH_EXTENSION_VALUES_DELETION, false);
enableIndexOnlyItems = configuration.getBoolean(PROPERTY_ENABLE_INDEX_ONLY_ITEMS, false);

textInfoColumnSize = configuration.getInt(PROPERTY_TEXT_INFO_COLUMN_SIZE, 255);
}

private boolean isAutoUpdate(String hbm2ddl) {
Expand Down Expand Up @@ -1000,6 +1006,10 @@ public boolean isEnableIndexOnlyItems() {
return enableIndexOnlyItems;
}

public int getTextInfoColumnSize() {
return textInfoColumnSize;
}

// for testing only
@SuppressWarnings("SameParameterValue")
public void setEnableNoFetchExtensionValuesInsertion(boolean enableNoFetchExtensionValuesInsertion) {
Expand Down
2 changes: 2 additions & 0 deletions repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryFactory.java
View file
Expand Up @@ -27,6 +27,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.Validate;
import org.h2.tools.Server;
import org.jetbrains.annotations.NotNull;

import java.io.File;
import java.io.IOException;
Expand All @@ -50,6 +51,7 @@ public class SqlRepositoryFactory implements RepositoryServiceFactory {

private SqlPerformanceMonitorImpl performanceMonitor;

@NotNull
public SqlRepositoryConfiguration getSqlConfiguration() {
Validate.notNull(sqlConfiguration, "Sql repository configuration not available (null).");
return sqlConfiguration;
Expand Down
6 changes: 5 additions & 1 deletion repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/data/RepositoryContext.java
View file
Expand Up @@ -18,6 +18,7 @@

import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.repo.api.RepositoryService;
import com.evolveum.midpoint.repo.sql.SqlRepositoryConfiguration;
import com.evolveum.midpoint.repo.sql.data.common.dictionary.ExtItemDictionary;
import com.evolveum.midpoint.schema.RelationRegistry;
import org.jetbrains.annotations.NotNull;
Expand All @@ -30,12 +31,15 @@ public class RepositoryContext {
@NotNull public final PrismContext prismContext;
@NotNull public final RelationRegistry relationRegistry;
@NotNull public final ExtItemDictionary extItemDictionary;
@NotNull public final SqlRepositoryConfiguration configuration;

public RepositoryContext(@NotNull RepositoryService repositoryService, @NotNull PrismContext prismContext,
@NotNull RelationRegistry relationRegistry, @NotNull ExtItemDictionary extItemDictionary) {
@NotNull RelationRegistry relationRegistry, @NotNull ExtItemDictionary extItemDictionary,
@NotNull SqlRepositoryConfiguration configuration) {
this.repositoryService = repositoryService;
this.prismContext = prismContext;
this.relationRegistry = relationRegistry;
this.extItemDictionary = extItemDictionary;
this.configuration = configuration;
}
}
21 changes: 12 additions & 9 deletions ...po-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/data/common/RObjectTextInfo.java
View file
Expand Up @@ -57,13 +57,14 @@ public class RObjectTextInfo implements Serializable {
public static final String COLUMN_OWNER_OID = "owner_oid";
public static final String F_TEXT = "text";

public static final int MAX_TEXT_SIZE = 255;
private static final int DEFAULT_MAX_TEXT_SIZE = 255;

private RObject owner;
private String ownerOid;

private String text;

@SuppressWarnings("unused")
public RObjectTextInfo() {
}

Expand Down Expand Up @@ -99,7 +100,7 @@ public void setOwnerOid(String ownerOid) {
}

@Id
@Column(name = "text", length = MAX_TEXT_SIZE)
@Column(name = "text", length = DEFAULT_MAX_TEXT_SIZE)
public String getText() {
return text;
}
Expand All @@ -124,7 +125,7 @@ public int hashCode() {
return Objects.hash(getOwnerOid(), getText());
}

public static <T extends ObjectType> Set<RObjectTextInfo> createItemsSet(@NotNull ObjectType object, @NotNull RObject repo,
public static Set<RObjectTextInfo> createItemsSet(@NotNull ObjectType object, @NotNull RObject repo,
@NotNull RepositoryContext repositoryContext) {

FullTextSearchConfigurationType config = repositoryContext.repositoryService.getFullTextSearchConfiguration();
Expand Down Expand Up @@ -165,16 +166,18 @@ public static <T extends ObjectType> Set<RObjectTextInfo> createItemsSet(@NotNul
}
}
}
LOGGER.trace("Indexing {}:\n items: {}\n values: {}\n words: {}", object, paths, values, allWords);
return createItemsSet(repo, allWords);
int maxTextSize = repositoryContext.configuration.getTextInfoColumnSize();
LOGGER.trace("Indexing {}:\n items: {}\n values: {}\n words: {}\n max text size: {}", object, paths, values,
allWords, maxTextSize);
return createItemsSet(repo, allWords, maxTextSize);
}

private static Set<RObjectTextInfo> createItemsSet(RObject repo, List<String> allWords) {
private static Set<RObjectTextInfo> createItemsSet(RObject repo, List<String> allWords, int maxTextSize) {
Set<RObjectTextInfo> rv = new HashSet<>();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < allWords.size(); i++) {
String word = allWords.get(i);
if (sb.length() + word.length() + 2 <= MAX_TEXT_SIZE) {
if (sb.length() + word.length() + 2 <= maxTextSize) {
sb.append(" ").append(word);
} else {
if (sb.length() > 0) {
Expand All @@ -185,8 +188,8 @@ private static Set<RObjectTextInfo> createItemsSet(RObject repo, List<String> al
} else {
// a problem - too large string
LOGGER.warn("Word too long to be correctly indexed: {}", word);
rv.add(new RObjectTextInfo(repo, " " + word.substring(0, MAX_TEXT_SIZE - 2) + " "));
allWords.set(i, word.substring(MAX_TEXT_SIZE - 2));
rv.add(new RObjectTextInfo(repo, " " + word.substring(0, maxTextSize - 2) + " "));
allWords.set(i, word.substring(maxTextSize - 2));
i--; // to reiterate (with shortened word)
}
}
Expand Down
1 change: 1 addition & 0 deletions repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/BaseHelper.java
View file
Expand Up @@ -107,6 +107,7 @@ public Session beginTransaction(boolean readOnly) {
return session;
}

@NotNull
public SqlRepositoryConfiguration getConfiguration() {
return repositoryFactory.getSqlConfiguration();
}
Expand Down
3 changes: 2 additions & 1 deletion ...ql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/CertificationCaseHelper.java
View file
Expand Up @@ -81,6 +81,7 @@ public class CertificationCaseHelper {
@Autowired private ObjectRetriever objectRetriever;
@Autowired private RepositoryService repositoryService;
@Autowired private ExtItemDictionary extItemDictionary;
@Autowired private BaseHelper baseHelper;

public void addCertificationCampaignCases(Session session, RObject object, boolean deleteBeforeAdd) {
if (!(object instanceof RAccessCertificationCampaign)) {
Expand Down Expand Up @@ -121,7 +122,7 @@ public void addCertificationCampaignCases(Session session, String campaignOid, C

@NotNull
private RepositoryContext createRepositoryContext() {
return new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary);
return new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary, baseHelper.getConfiguration());
}

public void deleteCertificationCampaignCases(Session session, String oid) {
Expand Down
16 changes: 9 additions & 7 deletions ...epo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/ObjectDeltaUpdater.java
View file
Expand Up @@ -18,7 +18,6 @@

import com.evolveum.midpoint.prism.*;
import com.evolveum.midpoint.prism.delta.ItemDelta;
import com.evolveum.midpoint.prism.delta.ItemDeltaCollectionsUtil;
import com.evolveum.midpoint.prism.path.ItemName;
import com.evolveum.midpoint.prism.path.ItemPath;
import com.evolveum.midpoint.repo.api.RepoModifyOptions;
Expand Down Expand Up @@ -83,6 +82,7 @@ public class ObjectDeltaUpdater {
@Autowired private RelationRegistry relationRegistry;
@Autowired private PrismEntityMapper prismEntityMapper;
@Autowired private ExtItemDictionary extItemDictionary;
@Autowired private BaseHelper baseHelper;

private static class Context {
private final RepoModifyOptions options;
Expand Down Expand Up @@ -250,7 +250,8 @@ private void handlePhoto(Object bean, ItemDelta delta) throws SchemaException {
}

MapperContext context = new MapperContext();
context.setRepositoryContext(new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary));
context.setRepositoryContext(new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary,
baseHelper.getConfiguration()));
context.setDelta(delta);
context.setOwner(bean);

Expand Down Expand Up @@ -285,7 +286,8 @@ private void handleWholeMetadata(Metadata<?> bean, ItemDelta delta) {
}

MapperContext context = new MapperContext();
context.setRepositoryContext(new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary));
context.setRepositoryContext(new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary,
baseHelper.getConfiguration()));
context.setDelta(delta);
context.setOwner(bean);

Expand Down Expand Up @@ -320,7 +322,8 @@ private void handleOperationResult(Object bean, ItemDelta delta) {
}

MapperContext context = new MapperContext();
context.setRepositoryContext(new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary));
context.setRepositoryContext(new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary,
baseHelper.getConfiguration()));
context.setDelta(delta);
context.setOwner(bean);

Expand Down Expand Up @@ -401,7 +404,7 @@ private void handleObjectTextInfoChanges(Class<? extends ObjectType> type, Colle
}

Set<RObjectTextInfo> newInfos = RObjectTextInfo.createItemsSet((ObjectType) prismObject.asObjectable(), object,
new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary));
new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary, baseHelper.getConfiguration()));

if (newInfos == null || newInfos.isEmpty()) {
object.getTextInfoItems().clear();
Expand Down Expand Up @@ -1022,8 +1025,7 @@ private Collection<PrismEntityPair> processDeltaValues(Collection<? extends Pris
for (PrismValue value : values) {
MapperContext context = new MapperContext();
context.setRepositoryContext(new RepositoryContext(repositoryService, prismContext, relationRegistry,
extItemDictionary
));
extItemDictionary, baseHelper.getConfiguration()));
context.setDelta(delta);
context.setOwner(bean);

Expand Down
2 changes: 1 addition & 1 deletion repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/ObjectUpdater.java
View file
Expand Up @@ -602,7 +602,7 @@ public <T extends ObjectType> RObject createDataObjectFromJAXB(PrismObject<T> pr
Method method = clazz.getMethod("copyFromJAXB", object.getClass(), clazz,
RepositoryContext.class, IdGeneratorResult.class);
method.invoke(clazz, object, rObject, new RepositoryContext(repositoryService, prismContext, relationRegistry,
extItemDictionary), generatorResult);
extItemDictionary, baseHelper.getConfiguration()), generatorResult);
} catch (Exception ex) {
SerializationRelatedException serializationException = ExceptionUtil.findCause(ex, SerializationRelatedException.class);
if (serializationException != null) {
Expand Down
5 changes: 4 additions & 1 deletion ...l-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/modify/PrismEntityMapper.java
View file
Expand Up @@ -28,6 +28,7 @@
import com.evolveum.midpoint.repo.sql.data.common.dictionary.ExtItemDictionary;
import com.evolveum.midpoint.repo.sql.data.common.embedded.*;
import com.evolveum.midpoint.repo.sql.data.common.enums.SchemaEnum;
import com.evolveum.midpoint.repo.sql.helpers.BaseHelper;
import com.evolveum.midpoint.repo.sql.helpers.mapper.*;
import com.evolveum.midpoint.schema.RelationRegistry;
import com.evolveum.midpoint.util.exception.SystemException;
Expand Down Expand Up @@ -74,6 +75,7 @@ public class PrismEntityMapper {
@Autowired private PrismContext prismContext;
@Autowired private ExtItemDictionary extItemDictionary;
@Autowired private RelationRegistry relationRegistry;
@Autowired private BaseHelper baseHelper;

public boolean supports(Class inputType, Class outputType) {
Key key = buildKey(inputType, outputType);
Expand Down Expand Up @@ -107,7 +109,8 @@ public <I, O> O map(I input, Class<O> outputType, MapperContext context) {
if (context == null) {
context = new MapperContext();
}
context.setRepositoryContext(new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary));
context.setRepositoryContext(new RepositoryContext(repositoryService, prismContext, relationRegistry, extItemDictionary,
baseHelper.getConfiguration()));

Key key = buildKey(input.getClass(), outputType);
Mapper<I, O> mapper = mappers.get(key);
Expand Down

0 comments on commit a64e9d7

Please sign in to comment.