Skip to content

Commit

Permalink
Fixing many many consistency issues. Some tests still failing. (MID-3…
Browse files Browse the repository at this point in the history 
…093, MID-2134)
  • Loading branch information
@semancik
semancik committed Jun 8, 2016
1 parent ed44fe9 commit 2bbe6e3
Show file tree
Hide file tree
Showing 11 changed files with 463 additions and 85 deletions.
42 changes: 38 additions & 4 deletions infra/schema/src/main/java/com/evolveum/midpoint/schema/GetOperationOptions.java
View file
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2010-2013 Evolveum
* Copyright (c) 2010-2016 Evolveum
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -415,9 +415,43 @@ public GetOperationOptions clone() {

@Override
public String toString() {
return "GetOperationOptions(resolve=" + resolve + ", resolveNames=" + resolveNames + ",noFetch=" + noFetch
+ ", raw=" + raw + ", doNotDiscovery="+doNotDiscovery+", retrieve="+retrieve+", allowNotFound="+ allowNotFound
+", relationalValueSearchQuery="+relationalValueSearchQuery+")";
StringBuilder sb = new StringBuilder("GetOperationOptions(");
appendFlag(sb, "resolve", resolve);
appendFlag(sb, "resolveNames", resolveNames);
appendFlag(sb, "noFetch", noFetch);
appendFlag(sb, "raw", raw);
appendFlag(sb, "doNotDiscovery", doNotDiscovery);
appendVal(sb, "retrieve", retrieve);
appendFlag(sb, "allowNotFound", allowNotFound);
appendVal(sb, "relationalValueSearchQuery", relationalValueSearchQuery);
if (sb.charAt(sb.length() - 1) == ',') {
sb.deleteCharAt(sb.length() - 1);
}
sb.append(")");
return sb.toString();
}

private void appendFlag(StringBuilder sb, String name, Boolean val) {
if (val == null) {
return;
} else if (val) {
sb.append(name);
sb.append(",");
} else {
sb.append(name);
sb.append("=false,");
}
}

private void appendVal(StringBuilder sb, String name, Object val) {
if (val == null) {
return;
} else {
sb.append(name);
sb.append("=");
sb.append(val);
sb.append(",");
}
}

}
7 changes: 7 additions & 0 deletions model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/LensElementContext.java
View file
Expand Up @@ -16,11 +16,18 @@
package com.evolveum.midpoint.model.impl.lens;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

import javax.xml.namespace.QName;

import com.evolveum.midpoint.prism.ConsistencyCheckScope;
import com.evolveum.midpoint.prism.Objectable;
import com.evolveum.midpoint.schema.DeltaConvertor;
import com.evolveum.midpoint.schema.constants.MidPointConstants;
import com.evolveum.midpoint.schema.constants.SchemaConstants;
import com.evolveum.midpoint.schema.processor.ResourceAttribute;
import com.evolveum.midpoint.schema.processor.ResourceAttributeContainer;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.util.exception.CommunicationException;
import com.evolveum.midpoint.util.exception.ConfigurationException;
Expand Down
25 changes: 25 additions & 0 deletions ...model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/LensProjectionContext.java
View file
Expand Up @@ -30,6 +30,7 @@
import com.evolveum.midpoint.prism.delta.ItemDelta;
import com.evolveum.midpoint.schema.DeltaConvertor;
import com.evolveum.midpoint.schema.ResourceShadowDiscriminator;
import com.evolveum.midpoint.schema.constants.SchemaConstants;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.util.exception.*;
import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
Expand All @@ -52,6 +53,7 @@
import com.evolveum.midpoint.prism.delta.ReferenceDelta;
import com.evolveum.midpoint.prism.path.ItemPath;
import com.evolveum.midpoint.schema.processor.ResourceAttribute;
import com.evolveum.midpoint.schema.processor.ResourceAttributeContainer;
import com.evolveum.midpoint.schema.processor.ResourceSchema;
import com.evolveum.midpoint.schema.util.MiscSchemaUtil;
import com.evolveum.midpoint.schema.util.ShadowUtil;
Expand Down Expand Up @@ -906,6 +908,29 @@ public void checkConsistence(String contextDesc, boolean fresh, boolean force) {
}
}

@Override
protected void checkConsistence(PrismObject<ShadowType> object, String elementDesc, String contextDesc) {
super.checkConsistence(object, elementDesc, contextDesc);
ResourceAttributeContainer attributesContainer = ShadowUtil.getAttributesContainer(object);
if (attributesContainer != null) {
ResourceType resource = getResource();
if (resource != null) {
String resourceNamespace = ResourceTypeUtil.getResourceNamespace(resource);
for(ResourceAttribute<?> attribute: attributesContainer.getAttributes()) {
QName attrName = attribute.getElementName();
if (SchemaConstants.NS_ICF_SCHEMA.equals(attrName.getNamespaceURI())) {
continue;
}
if (resourceNamespace.equals(attrName.getNamespaceURI())) {
continue;
}
String desc = elementDesc+" in "+this + (contextDesc == null ? "" : " in " +contextDesc);
throw new IllegalStateException("Invalid namespace for attribute "+attrName+" in "+desc);
}
}
}
}

protected boolean isRequireSecondardyDeltaOid() {
if (synchronizationPolicyDecision == SynchronizationPolicyDecision.ADD ||
synchronizationPolicyDecision == SynchronizationPolicyDecision.BROKEN ||
Expand Down
135 changes: 88 additions & 47 deletions ...del-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/projector/ContextLoader.java
View file
Expand Up @@ -29,6 +29,7 @@
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;

import com.evolveum.midpoint.common.InternalsConfig;
import com.evolveum.midpoint.common.refinery.RefinedObjectClassDefinition;
import com.evolveum.midpoint.model.api.ModelExecuteOptions;
import com.evolveum.midpoint.model.api.PolicyViolationException;
Expand Down Expand Up @@ -517,7 +518,7 @@ private <F extends FocusType> void loadLinkRefsFromFocus(LensContext<F> context,
}
LensProjectionContext accountContext = getOrCreateAccountContext(context, shadow, task, result);
accountContext.setFresh(true);
accountContext.setExists(true);
accountContext.setExists(shadow != null);
if (context.isDoReconciliationForAllProjections()) {
accountContext.setDoReconciliation(true);
}
Expand Down Expand Up @@ -720,57 +721,74 @@ private <F extends FocusType> void loadLinkRefsFromDelta(LensContext<F> context,
private <F extends ObjectType> void loadProjectionContextsSync(LensContext<F> context, Task task, OperationResult result) throws SchemaException,
ObjectNotFoundException, CommunicationException, ConfigurationException,
SecurityViolationException {
for (LensProjectionContext accountCtx : context.getProjectionContexts()) {
if (accountCtx.isFresh() && accountCtx.getObjectCurrent() != null) {
for (LensProjectionContext projCtx : context.getProjectionContexts()) {
if (projCtx.isFresh() && projCtx.getObjectCurrent() != null) {
// already loaded
continue;
}
ObjectDelta<ShadowType> syncDelta = accountCtx.getSyncDelta();
ObjectDelta<ShadowType> syncDelta = projCtx.getSyncDelta();
if (syncDelta != null) {
if (accountCtx.isDoReconciliation()) {
if (projCtx.isDoReconciliation()) {
// Do not load old account now. It will get loaded later in the
// reconciliation step. Just mark it as fresh.
accountCtx.setFresh(true);
projCtx.setFresh(true);
continue;
}
String oid = syncDelta.getOid();
PrismObject<ShadowType> account = null;
PrismObject<ShadowType> shadow = null;

if (syncDelta.getChangeType() == ChangeType.ADD) {
account = syncDelta.getObjectToAdd().clone();
accountCtx.setLoadedObject(account);
accountCtx.setExists(true);
shadow = syncDelta.getObjectToAdd().clone();
projCtx.setLoadedObject(shadow);
projCtx.setExists(true);

} else {

if (oid == null) {
throw new IllegalArgumentException("No OID in sync delta in " + accountCtx);
throw new IllegalArgumentException("No OID in sync delta in " + projCtx);
}
// Using NO_FETCH so we avoid reading in a full account. This is more efficient as we don't need full account here.
// We need to fetch from provisioning and not repository so the correct definition will be set.
Collection<SelectorOptions<GetOperationOptions>> options = SelectorOptions.createCollection(GetOperationOptions.createNoFetch());
account = provisioningService.getObject(ShadowType.class, oid, options, task, result);
GetOperationOptions option = GetOperationOptions.createNoFetch();
option.setDoNotDiscovery(true);
Collection<SelectorOptions<GetOperationOptions>> options = SelectorOptions.createCollection(option);

try {

shadow = provisioningService.getObject(ShadowType.class, oid, options, task, result);

} catch (ObjectNotFoundException e) {
LOGGER.trace("Loading shadow {} from sync delta failed: not found", oid);
projCtx.setExists(false);
projCtx.setObjectCurrent(null);
}

// We will not set old account if the delta is delete. The
// account does not really exists now.
// (but the OID and resource will be set from the repo
// shadow)
if (syncDelta.getChangeType() == ChangeType.DELETE) {
accountCtx.setExists(false);
} else {
syncDelta.applyTo(account);
accountCtx.setLoadedObject(account);
accountCtx.setExists(true);
projCtx.setExists(false);
projCtx.getResourceShadowDiscriminator().setThombstone(true);
} else if (shadow != null) {
syncDelta.applyTo(shadow);
projCtx.setLoadedObject(shadow);
projCtx.setExists(true);
}
}

// Make sure OID is set correctly
accountCtx.setOid(oid);
projCtx.setOid(oid);
// Make sure that resource is also resolved
if (accountCtx.getResource() == null) {
String resourceOid = ShadowUtil.getResourceOid(account.asObjectable());
if (projCtx.getResource() == null && shadow != null) {
String resourceOid = ShadowUtil.getResourceOid(shadow.asObjectable());
if (resourceOid == null) {
throw new IllegalArgumentException("No resource OID in " + account);
throw new IllegalArgumentException("No resource OID in " + shadow);
}
ResourceType resourceType = LensUtil.getResource(context, resourceOid, provisioningService, task, result);
accountCtx.setResource(resourceType);
projCtx.setResource(resourceType);
}
accountCtx.setFresh(true);
projCtx.setFresh(true);
}
}
}
Expand Down Expand Up @@ -924,6 +942,8 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
LensProjectionContext projContext, Task task, OperationResult result)
throws ObjectNotFoundException, CommunicationException, SchemaException, ConfigurationException,
SecurityViolationException {

String projectionHumanReadableName = projContext.getHumanReadableName();

if (projContext.getSynchronizationPolicyDecision() == SynchronizationPolicyDecision.BROKEN) {
return;
Expand All @@ -944,6 +964,7 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
}

// Load current object
boolean thombstone = false;
PrismObject<ShadowType> projectionObject = projContext.getObjectCurrent();
if (projContext.getObjectCurrent() == null || needToReload(context, projContext)) {
if (projContext.isAdd()) {
Expand All @@ -956,7 +977,7 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
projContext.setExists(false);
if (projContext.getResourceShadowDiscriminator() == null || projContext.getResourceShadowDiscriminator().getResourceOid() == null) {
throw new SystemException(
"Projection with null OID, no representation and no resource OID in account sync context "+projContext);
"Projection "+projectionHumanReadableName+" with null OID, no representation and no resource OID in account sync context "+projContext);
}
} else {
projContext.setExists(true);
Expand All @@ -972,14 +993,25 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
rootOptions.setAllowNotFound(true);
Collection<SelectorOptions<GetOperationOptions>> options = SelectorOptions.createCollection(rootOptions);
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("Loading projection shadow {}, options={}", projectionObjectOid, options);
LOGGER.trace("Loading shadow {} for projection {}, options={}", projectionObjectOid, projectionHumanReadableName, options);
}
try{

try {
PrismObject<ShadowType> objectOld = provisioningService.getObject(
projContext.getObjectTypeClass(), projectionObjectOid, options, task, result);
if (LOGGER.isTraceEnabled()) {
if (!GetOperationOptions.isNoFetch(rootOptions) && !GetOperationOptions.isRaw(rootOptions)) {
LOGGER.trace("Full shadow loaded: {}", objectOld);
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("Full shadow loaded for {}:\n{}", projectionHumanReadableName, objectOld.debugDump());
}
}
}
if (InternalsConfig.consistencyChecks) {
String resourceOid = projContext.getResourceOid();
if (resourceOid != null && !resourceOid.equals(objectOld.asObjectable().getResourceRef().getOid())) {
throw new IllegalStateException("Loaded shadow with wrong resourceRef. Loading shadow "+projectionObjectOid+", got "+
objectOld.getOid()+", expected resourceRef "+resourceOid+", but was "+objectOld.asObjectable().getResourceRef().getOid()+
" for context "+projectionHumanReadableName);
}
}
projContext.setLoadedObject(objectOld);
Expand All @@ -990,15 +1022,20 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
projContext.setFullShadow(false);
}
projectionObject = objectOld;
} catch (ObjectNotFoundException ex){
projContext.setSynchronizationPolicyDecision(SynchronizationPolicyDecision.BROKEN);
LOGGER.warn("Could not find object with oid " + projectionObjectOid + ". Context for these object is marked as broken");
return;
} catch (SchemaException ex){

} catch (ObjectNotFoundException ex) {
// This does not mean BROKEN. The projection was there, but it gone now. What we really want here
// is a thombstone projection.
thombstone = true;
projContext.setFullShadow(false);
LOGGER.warn("Could not find object with oid {}. The projection context {} is marked as thombstone.", projectionObjectOid, projectionHumanReadableName);

} catch (SchemaException ex) {
projContext.setSynchronizationPolicyDecision(SynchronizationPolicyDecision.BROKEN);
LOGGER.warn("Schema problem while getting object with oid " + projectionObjectOid + ". Context for these object is marked as broken");
LOGGER.warn("Schema problem while getting object with oid {}. Projection context {} is marked as broken", projectionObjectOid, projectionHumanReadableName);
return;
}

}
projContext.setFresh(true);
}
Expand All @@ -1020,7 +1057,7 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
} else if (projContext.getResourceShadowDiscriminator() != null) {
resourceOid = projContext.getResourceShadowDiscriminator().getResourceOid();
} else {
throw new IllegalStateException("No shadow and no resource intent means no resource OID in "+projContext);
throw new IllegalStateException("No shadow and no resource intent means no resource OID in "+projectionHumanReadableName);
}
} else {
resourceOid = resourceType.getOid();
Expand All @@ -1032,8 +1069,13 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
ShadowType accountShadowType = projectionObject.asObjectable();
String intent = ShadowUtil.getIntent(accountShadowType);
ShadowKindType kind = ShadowUtil.getKind(accountShadowType);
discr = new ResourceShadowDiscriminator(resourceOid, kind, intent);
discr = new ResourceShadowDiscriminator(resourceOid, kind, intent, thombstone);
projContext.setResourceShadowDiscriminator(discr);
} else {
if (thombstone) {
// We do not want to reset thombstone flag if it was set before
discr.setThombstone(thombstone);
}
}

// Load resource
Expand All @@ -1043,9 +1085,9 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
}

//Determine refined schema and password policies for account type
RefinedObjectClassDefinition rad = projContext.getStructuralObjectClassDefinition();
if (rad != null) {
ObjectReferenceType passwordPolicyRef = rad.getPasswordPolicy();
RefinedObjectClassDefinition structuralObjectClassDef = projContext.getStructuralObjectClassDefinition();
if (structuralObjectClassDef != null) {
ObjectReferenceType passwordPolicyRef = structuralObjectClassDef.getPasswordPolicy();
if (passwordPolicyRef != null && passwordPolicyRef.getOid() != null) {
PrismObject<ValuePolicyType> passwordPolicy = cacheRepositoryService.getObject(
ValuePolicyType.class, passwordPolicyRef.getOid(), null, result);
Expand All @@ -1065,6 +1107,8 @@ private <F extends ObjectType> void finishLoadOfProjectionContext(LensContext<F>
}

setPrimaryDeltaOldValue(projContext);

LOGGER.trace("FINISHED load of projection context {}\n{}", projectionHumanReadableName, projContext.debugDump());
}

private <F extends ObjectType> boolean needToReload(LensContext<F> context,
Expand Down Expand Up @@ -1197,14 +1241,11 @@ public <F extends ObjectType> void loadFullShadow(LensContext<F> context, LensPr

}

if (!compensated) {
projCtx.setSynchronizationPolicyDecision(SynchronizationPolicyDecision.BROKEN);
if (GetOperationOptions.isDoNotDiscovery(getOptions)) {
LOGGER.error("Load of full resource object {} resulted in ObjectNotFoundException (discovery disabled to avoid loops)", projCtx, getOptions);
throw ex;
} else {
// Setting the context to broken should be enough here.
}
if (!compensated) {
LOGGER.trace("ObjectNotFound error is not compensated, setting context to thombstone");
projCtx.getResourceShadowDiscriminator().setThombstone(true);
projCtx.setExists(false);
projCtx.setFullShadow(false);
}
}
}
Expand Down

0 comments on commit 2bbe6e3

Please sign in to comment.