From a2737b30e6507d871beb2ab020283b17c79c674b Mon Sep 17 00:00:00 2001 From: Radovan Semancik Date: Mon, 12 Feb 2018 17:22:11 +0100 Subject: [PATCH] Multi-resource propagation task (MID-4440) --- .../AbstractGroupingManualResourceTest.java | 4 +- .../intest/manual/TestSemiManualGrouping.java | 4 +- .../src/test/resources/logback-test.xml | 1 + .../manual/resource-semi-manual-grouping.xml | 7 +- ...rouping.xml => task-propagation-multi.xml} | 20 ++- .../src/test/resources/schema/resource.xsd | 42 ++++++ .../task/MultiPropagationResultHandler.java | 89 +++++++++++++ .../task/MultiPropagationTaskHandler.java | 125 ++++++++++++++++++ .../impl/task/PropagationTaskHandler.java | 12 +- .../AbstractSearchIterativeTaskHandler.java | 28 +++- 10 files changed, 315 insertions(+), 17 deletions(-) rename model/model-intest/src/test/resources/manual/{task-propagation-semi-manual-grouping.xml => task-propagation-multi.xml} (69%) create mode 100644 model/model-intest/src/test/resources/schema/resource.xsd create mode 100644 provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationResultHandler.java create mode 100644 provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationTaskHandler.java diff --git a/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractGroupingManualResourceTest.java b/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractGroupingManualResourceTest.java index b230cc698eb..ef136e3d834 100644 --- a/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractGroupingManualResourceTest.java +++ b/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/AbstractGroupingManualResourceTest.java @@ -141,8 +141,8 @@ public abstract class AbstractGroupingManualResourceTest extends AbstractManualR protected static final File TASK_PROPAGATION_MANUAL_GROUPING_FILE = new File(TEST_DIR, "task-propagation-manual-grouping.xml"); protected static final String TASK_PROPAGATION_MANUAL_GROUPING_OID = "b84a2c46-f0b5-11e7-baff-d35c2f14080f"; - protected static final File TASK_PROPAGATION_SEMI_MANUAL_GROUPING_FILE = new File(TEST_DIR, "task-propagation-semi-manual-grouping.xml"); - protected static final String TASK_PROPAGATION_SEMI_MANUAL_GROUPING_OID = "01db4542-f224-11e7-8833-bbe6634814e7"; + protected static final File TASK_PROPAGATION_MULTI_FILE = new File(TEST_DIR, "task-propagation-multi.xml"); + protected static final String TASK_PROPAGATION_MULTI_OID = "01db4542-f224-11e7-8833-bbe6634814e7"; private static final Trace LOGGER = TraceManager.getTrace(AbstractGroupingManualResourceTest.class); diff --git a/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/TestSemiManualGrouping.java b/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/TestSemiManualGrouping.java index 89c7c709974..f3bcfe6f6a0 100644 --- a/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/TestSemiManualGrouping.java +++ b/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/manual/TestSemiManualGrouping.java @@ -83,12 +83,12 @@ protected File getRoleTwoFile() { @Override protected String getPropagationTaskOid() { - return TASK_PROPAGATION_SEMI_MANUAL_GROUPING_OID; + return TASK_PROPAGATION_MULTI_OID; } @Override protected File getPropagationTaskFile() { - return TASK_PROPAGATION_SEMI_MANUAL_GROUPING_FILE; + return TASK_PROPAGATION_MULTI_FILE; } @Override diff --git a/model/model-intest/src/test/resources/logback-test.xml b/model/model-intest/src/test/resources/logback-test.xml index 2db0155f8b0..4de62521a44 100644 --- a/model/model-intest/src/test/resources/logback-test.xml +++ b/model/model-intest/src/test/resources/logback-test.xml @@ -83,6 +83,7 @@ + diff --git a/model/model-intest/src/test/resources/manual/resource-semi-manual-grouping.xml b/model/model-intest/src/test/resources/manual/resource-semi-manual-grouping.xml index 5b3d2d305d7..ecaf92fd76d 100644 --- a/model/model-intest/src/test/resources/manual/resource-semi-manual-grouping.xml +++ b/model/model-intest/src/test/resources/manual/resource-semi-manual-grouping.xml @@ -24,10 +24,15 @@ xmlns:icfc="http://midpoint.evolveum.com/xml/ns/public/connector/icf-1/connector-schema-3" xmlns:ri="http://midpoint.evolveum.com/xml/ns/public/resource/instance-3" xmlns:cap="http://midpoint.evolveum.com/xml/ns/public/resource/capabilities-3" - xmlns:q="http://prism.evolveum.com/xml/ns/public/query-3"> + xmlns:q="http://prism.evolveum.com/xml/ns/public/query-3" + xmlns:rext="http://midpoint.evolveum.com/xml/ns/samples/resource"> Semi-Manual Grouping CSV Resource + + propagated + + diff --git a/model/model-intest/src/test/resources/manual/task-propagation-semi-manual-grouping.xml b/model/model-intest/src/test/resources/manual/task-propagation-multi.xml similarity index 69% rename from model/model-intest/src/test/resources/manual/task-propagation-semi-manual-grouping.xml rename to model/model-intest/src/test/resources/manual/task-propagation-multi.xml index 7da996396f7..6e7e6b04966 100644 --- a/model/model-intest/src/test/resources/manual/task-propagation-semi-manual-grouping.xml +++ b/model/model-intest/src/test/resources/manual/task-propagation-multi.xml @@ -14,20 +14,32 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> + + + xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:q="http://prism.evolveum.com/xml/ns/public/query-3" + xmlns:rext="http://midpoint.evolveum.com/xml/ns/samples/resource"> - Propagation: Grouping Semi Manual Resource + Propagation: propagated provisioning resources 01db4542-f224-11e7-8833-bbe6634814e7 runnable - http://midpoint.evolveum.com/xml/ns/public/provisioning/task/propagation/handler-3 - + http://midpoint.evolveum.com/xml/ns/public/provisioning/task/propagation/multi-handler-3 + + + + extension/provisioning + propagated + + + + single diff --git a/model/model-intest/src/test/resources/schema/resource.xsd b/model/model-intest/src/test/resources/schema/resource.xsd new file mode 100644 index 00000000000..57194ec03b4 --- /dev/null +++ b/model/model-intest/src/test/resources/schema/resource.xsd @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + true + + + + + + + diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationResultHandler.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationResultHandler.java new file mode 100644 index 00000000000..e583c88f2c3 --- /dev/null +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationResultHandler.java @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2018 Evolveum + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.evolveum.midpoint.provisioning.impl.task; + +import com.evolveum.midpoint.prism.PrismObject; +import com.evolveum.midpoint.prism.query.ObjectFilter; +import com.evolveum.midpoint.prism.query.ObjectQuery; +import com.evolveum.midpoint.prism.query.builder.QueryBuilder; +import com.evolveum.midpoint.provisioning.impl.ShadowCache; +import com.evolveum.midpoint.provisioning.ucf.api.GenericFrameworkException; +import com.evolveum.midpoint.repo.api.PreconditionViolationException; +import com.evolveum.midpoint.repo.api.RepositoryService; +import com.evolveum.midpoint.repo.common.task.AbstractSearchIterativeResultHandler; +import com.evolveum.midpoint.schema.ResultHandler; +import com.evolveum.midpoint.schema.result.OperationResult; +import com.evolveum.midpoint.task.api.Task; +import com.evolveum.midpoint.task.api.TaskManager; +import com.evolveum.midpoint.util.exception.CommonException; +import com.evolveum.midpoint.util.exception.SystemException; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType; + +/** + * @author semancik + * + */ +public class MultiPropagationResultHandler extends AbstractSearchIterativeResultHandler { + + private static final transient Trace LOGGER = TraceManager.getTrace(MultiPropagationResultHandler.class); + + private final RepositoryService repositoryService; + private final ShadowCache shadowCache; + + public MultiPropagationResultHandler(Task coordinatorTask, String taskOperationPrefix, TaskManager taskManager, RepositoryService repositoryService, ShadowCache shadowCache) { + super(coordinatorTask, taskOperationPrefix, "propagation", "multipropagation", taskManager); + this.repositoryService = repositoryService; + this.shadowCache = shadowCache; + } + + @Override + protected boolean handleObject(PrismObject resource, Task workerTask, OperationResult taskResult) + throws CommonException, PreconditionViolationException { + + LOGGER.trace("Propagating provisioning operations on {}", resource); + ObjectQuery query = new ObjectQuery(); + ObjectFilter filter = QueryBuilder.queryFor(ShadowType.class, resource.getPrismContext()) + .item(ShadowType.F_RESOURCE_REF).ref(resource.getOid()) + .and() + .exists(ShadowType.F_PENDING_OPERATION) + .buildFilter(); + query.setFilter(filter); + + ResultHandler handler = + (shadow, result) -> { + propagateShadowOperations(resource, shadow, workerTask, result); + return true; + }; + + repositoryService.searchObjectsIterative(ShadowType.class, query, handler, null, false, taskResult); + + LOGGER.trace("Propagation of {} done", resource); + + return true; + } + + protected void propagateShadowOperations(PrismObject resource, PrismObject shadow, Task workerTask, OperationResult result) { + try { + shadowCache.propagateOperations(resource, shadow, workerTask, result); + } catch (CommonException | GenericFrameworkException e) { + throw new SystemException("Generic provisioning framework error: " + e.getMessage(), e); + } + } + +} diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationTaskHandler.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationTaskHandler.java new file mode 100644 index 00000000000..4d656e25591 --- /dev/null +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationTaskHandler.java @@ -0,0 +1,125 @@ +/** + * Copyright (c) 2018 Evolveum + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.evolveum.midpoint.provisioning.impl.task; + +import java.util.List; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.evolveum.midpoint.prism.PrismObject; +import com.evolveum.midpoint.prism.query.ObjectFilter; +import com.evolveum.midpoint.prism.query.ObjectQuery; +import com.evolveum.midpoint.prism.query.builder.QueryBuilder; +import com.evolveum.midpoint.provisioning.api.ProvisioningService; +import com.evolveum.midpoint.provisioning.impl.ShadowCache; +import com.evolveum.midpoint.provisioning.impl.ShadowCacheFactory; +import com.evolveum.midpoint.repo.common.task.AbstractSearchIterativeTaskHandler; +import com.evolveum.midpoint.schema.constants.SchemaConstants; +import com.evolveum.midpoint.schema.result.OperationConstants; +import com.evolveum.midpoint.schema.result.OperationResult; +import com.evolveum.midpoint.task.api.Task; +import com.evolveum.midpoint.task.api.TaskCategory; +import com.evolveum.midpoint.task.api.TaskManager; +import com.evolveum.midpoint.task.api.TaskRunResult; +import com.evolveum.midpoint.task.api.TaskRunResult.TaskRunResultStatus; +import com.evolveum.midpoint.util.exception.CommunicationException; +import com.evolveum.midpoint.util.exception.ConfigurationException; +import com.evolveum.midpoint.util.exception.ExpressionEvaluationException; +import com.evolveum.midpoint.util.exception.ObjectNotFoundException; +import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.util.exception.SecurityViolationException; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType; + +/** + * Task handler for provisioning propagation of many resources. + * + * The search in this task handler is somehow reversed. The task is searching for resources + * and then the task internally looks for pending changes. + * + * Here we assume that there will be large number of resources, but there will be much smaller + * number of changes. + * + * @author Radovan Semancik + * + */ +@Component +public class MultiPropagationTaskHandler extends AbstractSearchIterativeTaskHandler { + + public static final String HANDLER_URI = SchemaConstants.NS_PROVISIONING_TASK + "/propagation/multi-handler-3"; + + // WARNING! This task handler is efficiently singleton! + // It is a spring bean and it is supposed to handle all search task instances + // Therefore it must not have task-specific fields. It can only contain fields specific to + // all tasks of a specified type + + @Autowired private TaskManager taskManager; + @Autowired private ShadowCacheFactory shadowCacheFactory; + + private static final Trace LOGGER = TraceManager.getTrace(MultiPropagationTaskHandler.class); + + public MultiPropagationTaskHandler() { + super("Provisioning propagation (multi)", OperationConstants.PROVISIONING_PROPAGATION); + setLogFinishInfo(true); + setPreserveStatistics(false); + setEnableSynchronizationStatistics(false); + } + + @PostConstruct + private void initialize() { + taskManager.registerHandler(HANDLER_URI, this); + } + + @Override + protected MultiPropagationResultHandler createHandler(TaskRunResult runResult, Task coordinatorTask, + OperationResult opResult) { + + ShadowCache shadowCache = shadowCacheFactory.getShadowCache(ShadowCacheFactory.Mode.STANDARD); + MultiPropagationResultHandler handler = new MultiPropagationResultHandler(coordinatorTask, getTaskOperationPrefix(), taskManager, repositoryService, shadowCache); + return handler; + } + + @Override + public String getCategoryName(Task task) { + return TaskCategory.SYSTEM; + } + + @Override + public List getCategoryNames() { + return null; + } + + @Override + protected ObjectQuery createQuery(MultiPropagationResultHandler handler, TaskRunResult runResult, Task coordinatorTask, + OperationResult opResult) throws SchemaException { + ObjectQuery objectQuery = createQueryFromTask(handler, runResult, coordinatorTask, opResult); + LOGGER.trace("Resource query: {}", objectQuery); + return objectQuery; + + } + + @Override + protected Class getType(Task task) { + return ResourceType.class; + } + +} diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/PropagationTaskHandler.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/PropagationTaskHandler.java index a633f2da8ff..af0a4074e7b 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/PropagationTaskHandler.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/PropagationTaskHandler.java @@ -51,8 +51,11 @@ import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType; /** - * @author semancik - * + * Task handler for provisioning propagation of one resource. + * + * We assume that there will be few resources with a lot of changes each. + * + * @author Radovan Semancik */ @Component public class PropagationTaskHandler extends AbstractSearchIterativeTaskHandler { @@ -112,7 +115,7 @@ public List getCategoryNames() { } @Override - protected ObjectQuery createQuery(PropagationResultHandler handler, TaskRunResult runResult, Task task, + protected ObjectQuery createQuery(PropagationResultHandler handler, TaskRunResult runResult, Task coordinatorTask, OperationResult opResult) throws SchemaException { ObjectQuery query = new ObjectQuery(); ObjectFilter filter = QueryBuilder.queryFor(ShadowType.class, prismContext) @@ -120,9 +123,6 @@ protected ObjectQuery createQuery(PropagationResultHandler handler, TaskRunResul .and() .exists(ShadowType.F_PENDING_OPERATION) .buildFilter(); -// ObjectFilter filter = QueryBuilder.queryFor(ShadowType.class, prismContext) -// .exists(ShadowType.F_PENDING_OPERATION) -// .buildFilter(); query.setFilter(filter); return query; diff --git a/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/task/AbstractSearchIterativeTaskHandler.java b/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/task/AbstractSearchIterativeTaskHandler.java index 63332a09052..da282141b37 100644 --- a/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/task/AbstractSearchIterativeTaskHandler.java +++ b/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/task/AbstractSearchIterativeTaskHandler.java @@ -58,8 +58,10 @@ import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; import com.evolveum.midpoint.xml.ns._public.common.common_3.IterationMethodType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType; import com.evolveum.prism.xml.ns._public.query_3.QueryType; +import com.evolveum.prism.xml.ns._public.query_3.SearchFilterType; /** * @author semancik @@ -423,7 +425,7 @@ public void refreshStatus(Task task) { /** * Handler parameter may be used to pass task instance state between the calls. */ - protected abstract ObjectQuery createQuery(H handler, TaskRunResult runResult, Task task, OperationResult opResult) throws SchemaException; + protected abstract ObjectQuery createQuery(H handler, TaskRunResult runResult, Task coordinatorTask, OperationResult opResult) throws SchemaException; // useful e.g. to specify noFetch options for shadow-related queries protected Collection> createQueryOptions(H resultHandler, TaskRunResult runResult, Task coordinatorTask, OperationResult opResult) { @@ -476,7 +478,29 @@ protected ObjectQuery createQueryFromTaskIfExists(H handler, TaskRunResult runRe } protected QueryType getObjectQueryTypeFromTask(Task task) { - return getRealValue(task.getExtensionProperty(SchemaConstants.MODEL_EXTENSION_OBJECT_QUERY)); + QueryType queryType = getObjectQueryTypeFromTaskObjectRef(task); + if (queryType != null) { + return queryType; + } + return getObjectQueryTypeFromTaskExtension(task); + } + + protected QueryType getObjectQueryTypeFromTaskObjectRef(Task task) { + ObjectReferenceType objectRef = task.getObjectRef(); + if (objectRef == null) { + return null; + } + SearchFilterType filterType = objectRef.getFilter(); + if (filterType == null) { + return null; + } + QueryType queryType = new QueryType(); + queryType.setFilter(filterType); + return queryType; + } + + protected QueryType getObjectQueryTypeFromTaskExtension(Task task) { + return getRealValue(task.getExtensionProperty(SchemaConstants.MODEL_EXTENSION_OBJECT_QUERY)); } protected IterationMethodType getIterationMethodFromTask(Task task) {