Skip to content

Commit

Permalink
More work on attribute caching, getObject almost complete (MID-3481)
Browse files Browse the repository at this point in the history
  • Loading branch information
@semancik
semancik committed Oct 26, 2016
1 parent d259f95 commit 7f4c1db
Show file tree
Hide file tree
Showing 6 changed files with 365 additions and 35 deletions.
3 changes: 3 additions & 0 deletions infra/prism/src/main/java/com/evolveum/midpoint/prism/PrismProperty.java
View file
Expand Up @@ -394,6 +394,9 @@ public PropertyDelta<T> diff(PrismProperty<T> other, boolean ignoreMetadata, boo

public static <T> PropertyDelta<T> diff(PrismProperty<T> a, PrismProperty<T> b) {
if (a == null) {
if (b == null) {
return null;
}
PropertyDelta<T> delta = b.createDelta();
delta.addValuesToAdd(PrismValue.cloneCollection(b.getValues()));
return delta;
Expand Down
122 changes: 102 additions & 20 deletions infra/schema/src/main/java/com/evolveum/midpoint/schema/GetOperationOptions.java
View file
Expand Up @@ -106,7 +106,7 @@ public class GetOperationOptions implements Serializable, Cloneable {
* don't log it. In other cases, error in logs may lead to misleading
* information..
*/
Boolean allowNotFound;
private Boolean allowNotFound;

/**
* Return read-only object. The returned object will be only read by the client. The client will not modify it.
Expand All @@ -115,6 +115,14 @@ public class GetOperationOptions implements Serializable, Cloneable {
* at all times when the client do not plan to modify the returned object.
*/
private Boolean readOnly;

/**
* Requirement how stale or fresh the retrieved data should be. It specifies maximum age of the value in millisecods.
* The default value is zero, which means that a fresh value must always be returned. This means that caches that do
* not guarantee fresh value cannot be used. If non-zero value is specified then such caches may be used. In case that
* Long.MAX_VALUE is specified then the caches are always used and fresh value is never retrieved.
*/
private Long staleness;

public RetrieveOption getRetrieve() {
return retrieve;
Expand Down Expand Up @@ -394,6 +402,36 @@ public static boolean isReadOnly(GetOperationOptions options) {
return options.readOnly;
}

public Long getStaleness() {
return staleness;
}

public void setStaleness(Long staleness) {
this.staleness = staleness;
}

public static GetOperationOptions createStaleness(Long staleness) {
GetOperationOptions opts = new GetOperationOptions();
opts.setStaleness(staleness);
return opts;
}

public static GetOperationOptions createMaxStaleness() {
GetOperationOptions opts = new GetOperationOptions();
opts.setStaleness(Long.MAX_VALUE);
return opts;
}

public static long getStaleness(GetOperationOptions options) {
if (options == null) {
return 0L;
}
if (options.getStaleness() == null) {
return 0L;
}
return options.getStaleness();
}

public RelationalValueSearchQuery getRelationalValueSearchQuery() {
return relationalValueSearchQuery;
}
Expand All @@ -402,68 +440,110 @@ public void setRelationalValueSearchQuery(RelationalValueSearchQuery relationalV
this.relationalValueSearchQuery = relationalValueSearchQuery;
}

@Override
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((allowNotFound == null) ? 0 : allowNotFound.hashCode());
result = prime * result + ((doNotDiscovery == null) ? 0 : doNotDiscovery.hashCode());
result = prime * result + ((noFetch == null) ? 0 : noFetch.hashCode());
result = prime * result + ((raw == null) ? 0 : raw.hashCode());
result = prime * result + ((readOnly == null) ? 0 : readOnly.hashCode());
result = prime * result
+ ((relationalValueSearchQuery == null) ? 0 : relationalValueSearchQuery.hashCode());
result = prime * result + ((resolve == null) ? 0 : resolve.hashCode());
result = prime * result + ((resolveNames == null) ? 0 : resolveNames.hashCode());
result = prime * result + ((retrieve == null) ? 0 : retrieve.hashCode());
result = prime * result + ((staleness == null) ? 0 : staleness.hashCode());
result = prime * result + ((tolerateRawData == null) ? 0 : tolerateRawData.hashCode());
return result;
}

@Override
public boolean equals(Object obj) {
if (this == obj)
if (this == obj) {
return true;
if (obj == null)
}
if (obj == null) {
return false;
if (getClass() != obj.getClass())
}
if (getClass() != obj.getClass()) {
return false;
}
GetOperationOptions other = (GetOperationOptions) obj;
if (allowNotFound == null) {
if (other.allowNotFound != null)
if (other.allowNotFound != null) {
return false;
} else if (!allowNotFound.equals(other.allowNotFound))
}
} else if (!allowNotFound.equals(other.allowNotFound)) {
return false;
}
if (doNotDiscovery == null) {
if (other.doNotDiscovery != null)
if (other.doNotDiscovery != null) {
return false;
} else if (!doNotDiscovery.equals(other.doNotDiscovery))
}
} else if (!doNotDiscovery.equals(other.doNotDiscovery)) {
return false;
}
if (noFetch == null) {
if (other.noFetch != null)
if (other.noFetch != null) {
return false;
} else if (!noFetch.equals(other.noFetch))
}
} else if (!noFetch.equals(other.noFetch)) {
return false;
}
if (raw == null) {
if (other.raw != null)
if (other.raw != null) {
return false;
} else if (!raw.equals(other.raw))
}
} else if (!raw.equals(other.raw)) {
return false;
}
if (readOnly == null) {
if (other.readOnly != null) {
return false;
}
} else if (!readOnly.equals(other.readOnly)) {
return false;
}
if (relationalValueSearchQuery == null) {
if (other.relationalValueSearchQuery != null)
if (other.relationalValueSearchQuery != null) {
return false;
} else if (!relationalValueSearchQuery.equals(other.relationalValueSearchQuery))
}
} else if (!relationalValueSearchQuery.equals(other.relationalValueSearchQuery)) {
return false;
}
if (resolve == null) {
if (other.resolve != null)
if (other.resolve != null) {
return false;
} else if (!resolve.equals(other.resolve))
}
} else if (!resolve.equals(other.resolve)) {
return false;
}
if (resolveNames == null) {
if (other.resolveNames != null)
if (other.resolveNames != null) {
return false;
}
} else if (!resolveNames.equals(other.resolveNames)) {
return false;
}
if (retrieve != other.retrieve) {
return false;
}
if (staleness == null) {
if (other.staleness != null) {
return false;
} else if (!resolveNames.equals(other.resolveNames))
}
} else if (!staleness.equals(other.staleness)) {
return false;
if (retrieve != other.retrieve)
}
if (tolerateRawData == null) {
if (other.tolerateRawData != null) {
return false;
}
} else if (!tolerateRawData.equals(other.tolerateRawData)) {
return false;
}
return true;
}

Expand All @@ -477,6 +557,7 @@ public GetOperationOptions clone() {
clone.retrieve = this.retrieve;
clone.allowNotFound = this.allowNotFound;
clone.readOnly = this.readOnly;
clone.staleness = this.staleness;
if (this.relationalValueSearchQuery != null) {
clone.relationalValueSearchQuery = this.relationalValueSearchQuery.clone();
}
Expand All @@ -494,6 +575,7 @@ public String toString() {
appendVal(sb, "retrieve", retrieve);
appendFlag(sb, "allowNotFound", allowNotFound);
appendFlag(sb, "readOnly", readOnly);
appendVal(sb, "staleness", staleness);
appendVal(sb, "relationalValueSearchQuery", relationalValueSearchQuery);
if (sb.charAt(sb.length() - 1) == ',') {
sb.deleteCharAt(sb.length() - 1);
Expand Down
38 changes: 38 additions & 0 deletions .../provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowCache.java
View file
Expand Up @@ -20,6 +20,7 @@
import java.util.Iterator;
import java.util.List;

import javax.xml.datatype.XMLGregorianCalendar;
import javax.xml.namespace.QName;

import com.evolveum.midpoint.prism.Item;
Expand All @@ -30,6 +31,7 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;

import com.evolveum.midpoint.common.Clock;
import com.evolveum.midpoint.common.refinery.RefinedAssociationDefinition;
import com.evolveum.midpoint.common.refinery.RefinedAttributeDefinition;
import com.evolveum.midpoint.common.refinery.RefinedObjectClassDefinition;
Expand Down Expand Up @@ -68,6 +70,7 @@
import com.evolveum.midpoint.prism.query.SubstringFilter;
import com.evolveum.midpoint.prism.query.ValueFilter;
import com.evolveum.midpoint.prism.util.PrismUtil;
import com.evolveum.midpoint.prism.xml.XmlTypeConverter;
import com.evolveum.midpoint.provisioning.api.ChangeNotificationDispatcher;
import com.evolveum.midpoint.provisioning.api.GenericConnectorException;
import com.evolveum.midpoint.provisioning.api.ProvisioningOperationOptions;
Expand Down Expand Up @@ -119,7 +122,9 @@
import com.evolveum.midpoint.util.logging.TraceManager;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ActivationType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.AvailabilityStatusType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.CachingMetadataType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.FailedOperationTypeType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.MetadataType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationProvisioningScriptsType;
Expand Down Expand Up @@ -158,6 +163,9 @@ public abstract class ShadowCache {

@Autowired(required = true)
private ResourceManager resourceManager;

@Autowired(required = true)
private Clock clock;

@Autowired(required = true)
private PrismContext prismContext;
Expand Down Expand Up @@ -235,6 +243,11 @@ public PrismObject<ShadowType> getShadow(String oid, PrismObject<ShadowType> rep
parentResult.recordFatalError("Provided OID is not equal to OID of repository shadow");
throw new IllegalArgumentException("Provided OID is not equal to OID of repository shadow");
}

if (canReturnCached(options, repositoryShadow)) {
applyDefinition(repositoryShadow, parentResult);
return repositoryShadow;
}

ProvisioningContext ctx = ctxFactory.create(repositoryShadow, task, parentResult);
try {
Expand Down Expand Up @@ -347,6 +360,31 @@ public PrismObject<ShadowType> getShadow(String oid, PrismObject<ShadowType> rep

}

private boolean canReturnCached(Collection<SelectorOptions<GetOperationOptions>> options, PrismObject<ShadowType> repositoryShadow) throws ConfigurationException {
long stalenessOption = GetOperationOptions.getStaleness(SelectorOptions.findRootOptions(options));
if (stalenessOption == 0L) {
return false;
}
CachingMetadataType cachingMetadata = repositoryShadow.asObjectable().getCachingMetadata();
if (cachingMetadata == null) {
if (stalenessOption == Long.MAX_VALUE) {
// We must return cached version but there is no cached version.
throw new ConfigurationException("Cached version of "+repositoryShadow+" requested, but there is no cached value");
}
return false;
}
if (stalenessOption == Long.MAX_VALUE) {
return true;
}

XMLGregorianCalendar retrievalTimestamp = cachingMetadata.getRetrievalTimestamp();
if (retrievalTimestamp == null) {
return false;
}
long retrievalTimestampMillis = XmlTypeConverter.toMillis(retrievalTimestamp);
return (clock.currentTimeMillis() - retrievalTimestampMillis < stalenessOption);
}

private boolean isCompensate(GetOperationOptions rootOptions) {
return !GetOperationOptions.isDoNotDiscovery(rootOptions);
}
Expand Down
23 changes: 21 additions & 2 deletions ...rovisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java
View file
Expand Up @@ -814,6 +814,8 @@ public PrismObject<ShadowType> createRepositoryShadow(ProvisioningContext ctx, P

repoShadowType.setCachingMetadata(null);

ProvisioningUtil.cleanupShadowActivation(repoShadowType);

} else if (cachingStrategy == CachingStategyType.PASSIVE) {
// Do not need to clear anything. Just store all attributes and add metadata.
CachingMetadataType cachingMetadata = new CachingMetadataType();
Expand Down Expand Up @@ -857,8 +859,6 @@ public PrismObject<ShadowType> createRepositoryShadow(ProvisioningContext ctx, P

normalizeAttributes(repoShadow, ctx.getObjectClassDefinition());

ProvisioningUtil.cleanupShadowActivation(repoShadowType);

return repoShadow;
}

Expand Down Expand Up @@ -891,6 +891,9 @@ public Collection<ItemDelta> updateShadow(ProvisioningContext ctx, PrismObject<S
}
}
}

// TODO: reflect activation updates on cached shadow

if (LOGGER.isTraceEnabled()) {
LOGGER.trace("Updating repo shadow {}:\n{}", resourceShadow.getOid(), DebugUtil.debugDump(repoShadowChanges));
}
Expand Down Expand Up @@ -1011,6 +1014,12 @@ public PrismObject<ShadowType> updateShadow(ProvisioningContext ctx, PrismObject
}

} else if (cachingStrategy == CachingStategyType.PASSIVE) {

compareUpdateProperty(shadowDelta, SchemaConstants.PATH_ACTIVATION_ADMINISTRATIVE_STATUS, currentResourceShadow, oldRepoShadow);
compareUpdateProperty(shadowDelta, SchemaConstants.PATH_ACTIVATION_VALID_FROM, currentResourceShadow, oldRepoShadow);
compareUpdateProperty(shadowDelta, SchemaConstants.PATH_ACTIVATION_VALID_TO, currentResourceShadow, oldRepoShadow);
compareUpdateProperty(shadowDelta, SchemaConstants.PATH_ACTIVATION_LOCKOUT_STATUS, currentResourceShadow, oldRepoShadow);

CachingMetadataType cachingMetadata = new CachingMetadataType();
cachingMetadata.setRetrievalTimestamp(clock.currentTimeXMLGregorianCalendar());
shadowDelta.addModificationReplaceProperty(ShadowType.F_CACHING_METADATA, cachingMetadata);
Expand Down Expand Up @@ -1041,6 +1050,16 @@ public PrismObject<ShadowType> updateShadow(ProvisioningContext ctx, PrismObject
}
}

private <T> void compareUpdateProperty(ObjectDelta<ShadowType> shadowDelta,
ItemPath itemPath, PrismObject<ShadowType> currentResourceShadow, PrismObject<ShadowType> oldRepoShadow) {
PrismProperty<T> currentProperty = currentResourceShadow.findProperty(itemPath);
PrismProperty<T> oldProperty = oldRepoShadow.findProperty(itemPath);
PropertyDelta<T> itemDelta = PrismProperty.diff(oldProperty, currentProperty);
if (itemDelta != null && !itemDelta.isEmpty()) {
shadowDelta.addModification(itemDelta);
}
}

/**
* Re-reads the shadow, re-evaluates the identifiers and stored values, updates them if necessary. Returns
* fixed shadow.
Expand Down

0 comments on commit 7f4c1db

Please sign in to comment.