Skip to content

Commit

Permalink
repo-sqale: skeleton of extension attribute insertion, still WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
virgo47 committed Jun 7, 2021
1 parent df7fb99 commit b4e2d4b
Show file tree
Hide file tree
Showing 13 changed files with 306 additions and 10 deletions.
4 changes: 3 additions & 1 deletion repo/repo-sqale/sql/pgnew-repo.sql
Original file line number Diff line number Diff line change
Expand Up @@ -798,7 +798,8 @@ CREATE TABLE m_shadow (
primaryIdentifierValue TEXT,
-- status INTEGER, TODO how is this mapped? See RUtil.copyResultFromJAXB called from RTask and OperationResultMapper
synchronizationSituation SynchronizationSituationType,
synchronizationTimestamp TIMESTAMPTZ
synchronizationTimestamp TIMESTAMPTZ,
attributes JSONB
)
INHERITS (m_object);

Expand All @@ -816,6 +817,7 @@ ALTER TABLE m_shadow ADD CONSTRAINT m_shadow_nameNorm_key UNIQUE (nameNorm);
CREATE INDEX m_shadow_subtypes_idx ON m_shadow USING gin(subtypes);
CREATE INDEX m_shadow_policySituation_idx ON m_shadow USING GIN(policysituations gin__int_ops);
CREATE INDEX m_shadow_ext_idx ON m_shadow USING gin (ext);
CREATE INDEX m_shadow_attributes_idx ON m_shadow USING gin (attributes);
/*
TODO: reconsider, especially boolean things like dead (perhaps WHERE in other indexes?)
Also consider partitioning by some of the attributes (class/kind/intent?)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
/*
* Copyright (C) 2010-2021 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/
package com.evolveum.midpoint.repo.sqale;

import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;

import org.jetbrains.annotations.NotNull;

import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItem;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.QExtItem;
import com.evolveum.midpoint.repo.sqlbase.JdbcSession;
import com.evolveum.midpoint.util.logging.Trace;
import com.evolveum.midpoint.util.logging.TraceManager;

/**
* TODO
*/
public class ExtItemCache {

private static final Trace LOGGER = TraceManager.getTrace(ExtItemCache.class);

private final Map<Integer, MExtItem> idToExtItem = new ConcurrentHashMap<>();
private final Map<MExtItem.Key, MExtItem> keyToExtItem = new ConcurrentHashMap<>();

private Supplier<JdbcSession> jdbcSessionSupplier;

/**
* Initializes the ext-item cache.
* Provided {@link JdbcSession} supplier will be used for later writes as well.
*/
public synchronized void initialize(Supplier<JdbcSession> jdbcSessionSupplier) {
this.jdbcSessionSupplier = jdbcSessionSupplier;

// this can be called repeatedly in tests, so the clear may be necessary
idToExtItem.clear();
keyToExtItem.clear();

QExtItem uri = QExtItem.DEFAULT;
List<MExtItem> result;
try (JdbcSession jdbcSession = jdbcSessionSupplier.get().startReadOnlyTransaction()) {
result = jdbcSession.newQuery()
.select(uri)
.from(uri)
.fetch();
jdbcSession.commit();
}

for (MExtItem row : result) {
updateMaps(row);
}
LOGGER.info("Ext item cache initialized with {} items.", result.size());
}

private void updateMaps(MExtItem row) {
idToExtItem.put(row.id, row);
keyToExtItem.put(row.key(), row);
}

/*
public @Nullable Integer getId(@NotNull QName qName) {
return getId(QNameUtil.qNameToUri(qName));
}
public @NotNull Integer searchId(@NotNull QName qName) {
return searchId(QNameUtil.qNameToUri(qName));
}
public @NotNull Integer resolveUriToId(@NotNull QName qName) {
return resolveUriToId(QNameUtil.qNameToUri(qName));
}
public @Nullable Integer getId(@NotNull String uri) {
Integer id = extItemKeyToId.get(uri);
LOGGER.trace("URI cache 'get' returned ID={} for URI={}", id, uri);
return id;
}
public @NotNull Integer searchId(@NotNull String uri) {
Integer id = extItemKeyToId.getOrDefault(uri, UNKNOWN_ID);
LOGGER.trace("URI cache 'search' returned ID={} for URI={}", id, uri);
return id;
}
public @NotNull Integer resolveUriToId(@NotNull String uri) {
Integer id = getId(uri);
LOGGER.trace("URI cache 'resolve' returned ID={} for URI={}", id, uri);
return Objects.requireNonNull(id, () -> "URI not cached: " + uri);
}
public String getUri(Integer id) {
String uri = idToExtItem.get(id);
LOGGER.trace("URI cache 'get' returned URI={} for ID={}", uri, id);
return uri;
}
public @NotNull String resolveToUri(Integer id) {
String uri = idToExtItem.get(id);
LOGGER.trace("URI cache 'resolve' returned URI={} for ID={}", uri, id);
return Objects.requireNonNull(uri, () -> "No URI cached under ID " + id);
}
*/

public synchronized @NotNull MExtItem resolveExtensionItem(@NotNull MExtItem.Key extItemKey) {
if (jdbcSessionSupplier == null) {
throw new IllegalStateException("Ext item cache was not initialized yet!");
}

MExtItem extItem = keyToExtItem.get(extItemKey);
if (extItem != null) {
return extItem;
}

QExtItem ei = QExtItem.DEFAULT;
try (JdbcSession jdbcSession = jdbcSessionSupplier.get().startTransaction()) {
Integer id = jdbcSession.newInsert(ei)
.set(ei.itemName, extItemKey.itemName)
.set(ei.valueType, extItemKey.valueType)
.set(ei.holderType, extItemKey.holderType)
.set(ei.cardinality, extItemKey.cardinality)
.executeWithKey(ei.id);
jdbcSession.commit();

extItem = MExtItem.of(id, extItemKey);
updateMaps(extItem);
}

LOGGER.debug("Ext item cache row inserted: {}", extItem);
return extItem;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@

import com.evolveum.midpoint.repo.sqale.qmodel.common.MContainerType;
import com.evolveum.midpoint.repo.sqale.qmodel.common.QUri;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItem;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItemCardinality;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItemHolderType;
import com.evolveum.midpoint.repo.sqale.qmodel.object.MObjectType;
import com.evolveum.midpoint.repo.sqale.qmodel.ref.MReferenceType;
import com.evolveum.midpoint.repo.sqlbase.JdbcRepositoryConfiguration;
Expand All @@ -31,6 +34,7 @@
public class SqaleRepoContext extends SqlRepoContext {

private final UriCache uriCache;
private final ExtItemCache extItemCache;

public SqaleRepoContext(
JdbcRepositoryConfiguration jdbcRepositoryConfiguration,
Expand All @@ -43,6 +47,8 @@ public SqaleRepoContext(
querydslConfig.register(new EnumAsObjectType<>(ActivationStatusType.class));
querydslConfig.register(new EnumAsObjectType<>(AvailabilityStatusType.class));
querydslConfig.register(new EnumAsObjectType<>(MContainerType.class));
querydslConfig.register(new EnumAsObjectType<>(MExtItemHolderType.class));
querydslConfig.register(new EnumAsObjectType<>(MExtItemCardinality.class));
querydslConfig.register(new EnumAsObjectType<>(MObjectType.class));
querydslConfig.register(new EnumAsObjectType<>(MReferenceType.class));
querydslConfig.register(new EnumAsObjectType<>(LockoutStatusType.class));
Expand All @@ -63,12 +69,14 @@ public SqaleRepoContext(
querydslConfig.register(new QuerydslJsonbType());

uriCache = new UriCache();
extItemCache = new ExtItemCache();
}

// This has nothing to do with "repo cache" which is higher than this.
@PostConstruct
public void clearCaches() {
uriCache.initialize(this::newJdbcSession);
extItemCache.initialize(this::newJdbcSession);
}

/** @see UriCache#searchId(String) */
Expand Down Expand Up @@ -99,4 +107,8 @@ public Integer processCacheableRelation(QName qName) {
return processCacheableUri(
QNameUtil.qNameToUri(normalizeRelation(qName)));
}

public MExtItem resolveExtensionItem(MExtItem.Key extItemKey) {
return extItemCache.resolveExtensionItem(extItemKey);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -177,9 +177,11 @@ public String getUri(Integer id) {
id = jdbcSession.newInsert(qu)
.set(qu.uri, uri)
.executeWithKey(qu.id);
updateMaps(MUri.of(id, uri));
jdbcSession.commit();

updateMaps(MUri.of(id, uri));
}
// TODO query when constraint violation

LOGGER.debug("URI cache inserted URI={} under ID={}", uri, id);
return id;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@
*/
package com.evolveum.midpoint.repo.sqale.qmodel;

import java.util.Collection;
import java.util.List;
import java.util.UUID;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Function;
import javax.xml.namespace.QName;
Expand All @@ -19,13 +18,20 @@
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import com.evolveum.midpoint.prism.Containerable;
import com.evolveum.midpoint.prism.Item;
import com.evolveum.midpoint.prism.ItemDefinition;
import com.evolveum.midpoint.prism.PrismContainerValue;
import com.evolveum.midpoint.repo.sqale.SqaleRepoContext;
import com.evolveum.midpoint.repo.sqale.delta.item.*;
import com.evolveum.midpoint.repo.sqale.filtering.ArrayPathItemFilterProcessor;
import com.evolveum.midpoint.repo.sqale.filtering.RefItemFilterProcessor;
import com.evolveum.midpoint.repo.sqale.filtering.UriItemFilterProcessor;
import com.evolveum.midpoint.repo.sqale.mapping.SqaleItemSqlMapper;
import com.evolveum.midpoint.repo.sqale.qmodel.common.QUri;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItem;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItemCardinality;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItemHolderType;
import com.evolveum.midpoint.repo.sqale.qmodel.object.MObjectType;
import com.evolveum.midpoint.repo.sqale.qmodel.object.QObject;
import com.evolveum.midpoint.repo.sqale.qmodel.ref.MReference;
Expand All @@ -38,12 +44,14 @@
import com.evolveum.midpoint.repo.sqlbase.mapping.QueryModelMappingRegistry;
import com.evolveum.midpoint.repo.sqlbase.mapping.QueryTableMapping;
import com.evolveum.midpoint.repo.sqlbase.querydsl.FlexibleRelationalPathBase;
import com.evolveum.midpoint.repo.sqlbase.querydsl.Jsonb;
import com.evolveum.midpoint.repo.sqlbase.querydsl.UuidPath;
import com.evolveum.midpoint.schema.GetOperationOptions;
import com.evolveum.midpoint.schema.SelectorOptions;
import com.evolveum.midpoint.util.MiscUtil;
import com.evolveum.midpoint.util.QNameUtil;
import com.evolveum.midpoint.util.exception.SchemaException;
import com.evolveum.midpoint.util.exception.SystemException;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType;
import com.evolveum.prism.xml.ns._public.types_3.PolyStringType;

Expand Down Expand Up @@ -382,4 +390,50 @@ protected void insert(R row, JdbcSession jdbcSession) {
.populate(row)
.execute();
}

protected Jsonb processExtensions(Containerable extContainer, MExtItemHolderType holderType) {
if (extContainer == null) {
return null;
}

// TODO
// copyExtensionOrAttributesFromJAXB(jaxb.getAttributes().asPrismContainerValue(), repo, repositoryContext, RObjectExtensionType.ATTRIBUTES, generatorResult);
Map<String, Object> extMap = new LinkedHashMap<>();
PrismContainerValue<?> prismContainerValue = extContainer.asPrismContainerValue();
for (Item<?, ?> item : prismContainerValue.getItems()) {
MExtItem extItem = findExtensionItem(item, holderType);
extMap.put(extItem.id.toString(), extItemValue(item, holderType));
// Set<RAnyValue<?>> converted = converter.convertToRValue(item, false, ownerType);
// if (generatorResult.isGeneratedOid()) {
// converted.forEach(v -> v.setTransient(true));
// }
// values.addAll(converted);
}

try {
return Jsonb.from(extMap);
} catch (IOException e) {
throw new SystemException(e);
}
}

private MExtItem findExtensionItem(Item<?, ?> item, MExtItemHolderType holderType) {
Objects.requireNonNull(item, "Object for converting must not be null.");

ItemDefinition<?> definition = item.getDefinition();
Objects.requireNonNull(definition, "Item '" + item.getElementName() + "' without definition can't be saved.");

MExtItem.Key extItemKey = new MExtItem.Key();
extItemKey.itemName = QNameUtil.qNameToUri(definition.getItemName());
extItemKey.valueType = QNameUtil.qNameToUri(definition.getTypeName());
extItemKey.cardinality = definition.getMaxOccurs() == 1
? MExtItemCardinality.SCALAR : MExtItemCardinality.ARRAY;
extItemKey.holderType = holderType;

return repositoryContext().resolveExtensionItem(extItemKey);
}

private Object extItemValue(Item<?, ?> item, MExtItemHolderType holderType) {
return "VAL"; // TODO
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import com.evolveum.midpoint.repo.sqale.SqaleRepoContext;
import com.evolveum.midpoint.repo.sqale.qmodel.common.MContainerType;
import com.evolveum.midpoint.repo.sqale.qmodel.common.QContainerMapping;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItemHolderType;
import com.evolveum.midpoint.repo.sqale.qmodel.object.MObject;
import com.evolveum.midpoint.repo.sqlbase.JdbcSession;
import com.evolveum.midpoint.util.MiscUtil;
Expand Down Expand Up @@ -198,7 +199,7 @@ public MAssignment insert(AssignmentType assignment, OR ownerRow, JdbcSession jd
// row.extId = assignment.getExtension()...id?;
// row.extOid =;
row.policySituations = processCacheableUris(assignment.getPolicySituation());
// TODO extensions stored inline (JSON)
row.ext = processExtensions(assignment.getExtension(), MExtItemHolderType.EXTENSION);

ConstructionType construction = assignment.getConstruction();
if (construction != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.evolveum.midpoint.repo.sqale.SqaleUtils;
import com.evolveum.midpoint.repo.sqale.qmodel.SqaleTableMapping;
import com.evolveum.midpoint.repo.sqale.qmodel.common.QUri;
import com.evolveum.midpoint.repo.sqale.qmodel.ext.MExtItemHolderType;
import com.evolveum.midpoint.repo.sqale.qmodel.ref.QObjectReferenceMapping;
import com.evolveum.midpoint.repo.sqlbase.JdbcSession;
import com.evolveum.midpoint.repo.sqlbase.RepositoryObjectParseResult;
Expand Down Expand Up @@ -196,7 +197,7 @@ public R toRowObjectWithoutFullObject(S schemaObject, JdbcSession jdbcSession) {
row.subtypes = listToArray(schemaObject.getSubtype());
// TODO textInfo (fulltext support)
// repo.getTextInfoItems().addAll(RObjectTextInfo.createItemsSet(jaxb, repo, repositoryContext));
// TODO extensions stored inline (JSON) - that is ext column
row.ext = processExtensions(schemaObject.getExtension(), MExtItemHolderType.EXTENSION);

// This is duplicate code with QAssignmentMapping.insert, but making interface
// and needed setters (fields are not "interface-able") would create much more code.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import com.evolveum.midpoint.repo.sqale.qmodel.object.MObject;
import com.evolveum.midpoint.repo.sqale.qmodel.object.MObjectType;
import com.evolveum.midpoint.repo.sqlbase.querydsl.Jsonb;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowKindType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.SynchronizationSituationType;

Expand All @@ -33,4 +34,5 @@ public class MShadow extends MObject {
public String primaryIdentifierValue;
public SynchronizationSituationType synchronizationSituation;
public Instant synchronizationTimestamp;
public Jsonb attributes;
}

0 comments on commit b4e2d4b

Please sign in to comment.