From 297a355e975e870a6b223a5d6b992908c8a68ec3 Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Wed, 12 Sep 2018 12:32:51 +0200 Subject: [PATCH] Make strictly seq. paging the default (MID-4414) As a workaround for obsolete data processing described in MID-4414 we made the strictly sequential version of iteration-by-paging the default one. (It can be overridden in config.xml and/or in each individual searchObjectsIteratively call.) --- .../midpoint/prism/query/ObjectPaging.java | 14 +- .../AbstractSearchExpressionEvaluator.java | 17 +-- .../script/ScriptExpressionFactory.java | 2 +- .../model/impl/ModelObjectResolver.java | 4 +- .../midpoint/model/impl/ModelRestService.java | 16 +-- .../impl/controller/ModelController.java | 2 +- .../focus/ObjectTemplateProcessor.java | 2 +- .../test/AbstractModelIntegrationTest.java | 35 ++--- .../impl/ProvisioningServiceImpl.java | 2 +- .../provisioning/impl/ShadowManager.java | 2 +- .../ObjectAlreadyExistHandler.java | 5 +- .../task/MultiPropagationResultHandler.java | 5 +- .../provisioning/impl/dummy/TestDummy.java | 2 +- .../provisioning/impl/opendj/TestOpenDj.java | 11 +- .../midpoint/repo/api/RepositoryService.java | 32 ++++- .../midpoint/repo/cache/RepositoryCache.java | 9 +- .../repo/common/RepoObjectResolver.java | 27 ++-- .../AbstractSearchIterativeTaskHandler.java | 4 +- .../midpoint/repo/sql/AddGetObjectTest.java | 2 +- .../midpoint/repo/sql/ConcurrencyTest.java | 2 +- .../repo/sql/SearchIterativeTest.java | 123 +++++++++--------- .../repo/sql/ObjectPagingAfterOid.java | 4 +- .../repo/sql/SqlRepositoryConfiguration.java | 11 +- .../repo/sql/SqlRepositoryServiceImpl.java | 44 ++++--- .../repo/sql/helpers/ObjectRetriever.java | 34 +++-- .../repo/sql/helpers/OrgClosureManager.java | 2 +- .../test/AbstractIntegrationTest.java | 11 +- .../test/RepoSimpleObjectResolver.java | 20 +-- .../ninja/action/DeleteRepositoryAction.java | 4 +- .../action/worker/ExportProducerWorker.java | 2 +- 30 files changed, 228 insertions(+), 222 deletions(-) diff --git a/infra/prism/src/main/java/com/evolveum/midpoint/prism/query/ObjectPaging.java b/infra/prism/src/main/java/com/evolveum/midpoint/prism/query/ObjectPaging.java index abd14154b16..f565accb9ac 100644 --- a/infra/prism/src/main/java/com/evolveum/midpoint/prism/query/ObjectPaging.java +++ b/infra/prism/src/main/java/com/evolveum/midpoint/prism/query/ObjectPaging.java @@ -40,33 +40,33 @@ public class ObjectPaging implements DebugDumpable, Serializable { protected ObjectPaging() { } - ObjectPaging(Integer offset, Integer maxSize) { + private ObjectPaging(Integer offset, Integer maxSize) { this.offset = offset; this.maxSize = maxSize; } - ObjectPaging(Integer offset, Integer maxSize, ItemPath groupBy) { + private ObjectPaging(Integer offset, Integer maxSize, ItemPath groupBy) { this.offset = offset; this.maxSize = maxSize; setGrouping(groupBy); } - ObjectPaging(ItemPath orderBy, OrderDirection direction) { + private ObjectPaging(ItemPath orderBy, OrderDirection direction) { setOrdering(orderBy, direction); } - ObjectPaging(ItemPath orderBy, OrderDirection direction, ItemPath groupBy) { + private ObjectPaging(ItemPath orderBy, OrderDirection direction, ItemPath groupBy) { setOrdering(orderBy, direction); setGrouping(groupBy); } - ObjectPaging(Integer offset, Integer maxSize, ItemPath orderBy, OrderDirection direction) { + private ObjectPaging(Integer offset, Integer maxSize, ItemPath orderBy, OrderDirection direction) { this.offset = offset; this.maxSize = maxSize; setOrdering(orderBy, direction); } - ObjectPaging(Integer offset, Integer maxSize, ItemPath orderBy, OrderDirection direction, ItemPath groupBy) { + private ObjectPaging(Integer offset, Integer maxSize, ItemPath orderBy, OrderDirection direction, ItemPath groupBy) { this.offset = offset; this.maxSize = maxSize; setOrdering(orderBy, direction); @@ -74,7 +74,7 @@ protected ObjectPaging() { setGrouping(groupBy); } - ObjectPaging(ItemPath groupBy) { + private ObjectPaging(ItemPath groupBy) { setGrouping(groupBy); } diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/evaluator/AbstractSearchExpressionEvaluator.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/evaluator/AbstractSearchExpressionEvaluator.java index f6f61dbc22c..af87c324870 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/evaluator/AbstractSearchExpressionEvaluator.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/evaluator/AbstractSearchExpressionEvaluator.java @@ -309,18 +309,15 @@ private List executeSearchAttempt(final List handler = new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult parentResult) { - if (rawResult != null) { - rawResult.add(object); - } - list.add(createPrismValue(object.getOid(), targetTypeQName, additionalAttributeDeltas, params)); + ResultHandler handler = (object, parentResult) -> { + if (rawResult != null) { + rawResult.add(object); + } + list.add(createPrismValue(object.getOid(), targetTypeQName, additionalAttributeDeltas, params)); - // TODO: we should count results and stop after some reasonably high number? + // TODO: we should count results and stop after some reasonably high number? - return true; - } + return true; }; try { diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/script/ScriptExpressionFactory.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/script/ScriptExpressionFactory.java index cf6ce41ce92..b6c08f88528 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/script/ScriptExpressionFactory.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/script/ScriptExpressionFactory.java @@ -149,7 +149,7 @@ private synchronized void initializeCustomFunctionsLibraryCache(ExpressionFactor }; try { repositoryService.searchObjectsIterative(FunctionLibraryType.class, null, functionLibraryHandler, - SelectorOptions.createCollection(GetOperationOptions.createReadOnly()), false, subResult); + SelectorOptions.createCollection(GetOperationOptions.createReadOnly()), true, subResult); subResult.recordSuccessIfUnknown(); } catch (SchemaException | RuntimeException e) { subResult.recordFatalError("Failed to initialize custom functions", e); diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/ModelObjectResolver.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/ModelObjectResolver.java index faa538b41fc..47a930053b4 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/ModelObjectResolver.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/ModelObjectResolver.java @@ -194,9 +194,9 @@ public T getObject(Class clazz, String oid, Collection public void searchIterative(Class type, ObjectQuery query, Collection> options, ResultHandler handler, Task task, OperationResult parentResult) throws SchemaException, ObjectNotFoundException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { if (ObjectTypes.isClassManagedByProvisioning(type)) { - provisioning.searchObjectsIterative(type, query, options, handler, (Task) task, parentResult); + provisioning.searchObjectsIterative(type, query, options, handler, task, parentResult); } else { - cacheRepositoryService.searchObjectsIterative(type, query, handler, options, false, parentResult); // TODO pull up into resolver interface + cacheRepositoryService.searchObjectsIterative(type, query, handler, options, true, parentResult); // TODO pull up into resolver interface } } diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/ModelRestService.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/ModelRestService.java index 37b4543d9c7..f10644f3947 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/ModelRestService.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/ModelRestService.java @@ -35,11 +35,7 @@ import com.evolveum.midpoint.prism.query.QueryJaxbConvertor; import com.evolveum.midpoint.prism.query.builder.QueryBuilder; import com.evolveum.midpoint.repo.api.CacheDispatcher; -import com.evolveum.midpoint.schema.DefinitionProcessingOption; -import com.evolveum.midpoint.schema.DeltaConvertor; -import com.evolveum.midpoint.schema.GetOperationOptions; -import com.evolveum.midpoint.schema.ResultHandler; -import com.evolveum.midpoint.schema.SelectorOptions; +import com.evolveum.midpoint.schema.*; import com.evolveum.midpoint.schema.constants.MidPointConstants; import com.evolveum.midpoint.schema.constants.ObjectTypes; import com.evolveum.midpoint.schema.result.OperationResult; @@ -462,13 +458,11 @@ public Response searchObjectsByType(@PathParam("type") St Collection> searchOptions = GetOperationOptions.fromRestOptions(options, include, exclude, DefinitionProcessingOption.ONLY_IF_EXISTS); - List objects = new ArrayList<>(); - ResultHandler handler = (object, parentResult1) -> objects.add(object.asObjectable()); - - modelService.searchObjectsIterative(clazz, null, handler, searchOptions, task, parentResult); - + List> objects = modelService.searchObjects(clazz, null, searchOptions, task, parentResult); ObjectListType listType = new ObjectListType(); - listType.getObject().addAll(objects); + for (PrismObject object : objects) { + listType.getObject().add(object.asObjectable()); + } response = RestServiceUtil.createResponse(Response.Status.OK, listType, parentResult, true); // response = Response.ok().entity(listType).build(); diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java index 938ec9c2f49..6db2ef2398a 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java @@ -1138,7 +1138,7 @@ public SearchResultMetadata searchObjectsIterative(Class< try { switch (searchProvider) { - case REPOSITORY: metadata = cacheRepositoryService.searchObjectsIterative(type, query, internalHandler, options, false, result); break; // TODO move strictSequential flag to model API in some form + case REPOSITORY: metadata = cacheRepositoryService.searchObjectsIterative(type, query, internalHandler, options, true, result); break; case PROVISIONING: metadata = provisioning.searchObjectsIterative(type, query, options, internalHandler, task, result); break; case TASK_MANAGER: metadata = taskManager.searchObjectsIterative(type, query, options, internalHandler, result); break; default: throw new AssertionError("Unexpected search provider: " + searchProvider); diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/projector/focus/ObjectTemplateProcessor.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/projector/focus/ObjectTemplateProcessor.java index 886d2ffc15b..65b1c4471d7 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/projector/focus/ObjectTemplateProcessor.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/lens/projector/focus/ObjectTemplateProcessor.java @@ -493,7 +493,7 @@ private void collectAutoassignMapping } return true; }; - cacheRepositoryService.searchObjectsIterative(AbstractRoleType.class, query, handler, GetOperationOptions.createReadOnlyCollection(), false, result); + cacheRepositoryService.searchObjectsIterative(AbstractRoleType.class, query, handler, GetOperationOptions.createReadOnlyCollection(), true, result); } private void setMappingTarget(MappingType mapping, ItemPathType path) { diff --git a/model/model-test/src/main/java/com/evolveum/midpoint/model/test/AbstractModelIntegrationTest.java b/model/model-test/src/main/java/com/evolveum/midpoint/model/test/AbstractModelIntegrationTest.java index ef2bdb0e0bc..072346fed5a 100644 --- a/model/model-test/src/main/java/com/evolveum/midpoint/model/test/AbstractModelIntegrationTest.java +++ b/model/model-test/src/main/java/com/evolveum/midpoint/model/test/AbstractModelIntegrationTest.java @@ -497,16 +497,13 @@ protected void searchObjectsIterative(Class type, Obje OperationResult result = task.getResult(); final MutableInt count = new MutableInt(0); // Cannot convert to lambda here. Java does not want to understand the generic types properly. - SearchResultMetadata searchMetadata = modelService.searchObjectsIterative(type, query, new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult oresult) { - count.increment(); - if (handler != null) { - handler.accept(object); - } - return true; - } - }, null, task, result); + SearchResultMetadata searchMetadata = modelService.searchObjectsIterative(type, query, (object, oresult) -> { + count.increment(); + if (handler != null) { + handler.accept(object); + } + return true; + }, null, task, result); if (verbose) display(type.getSimpleName()+"s", count.getValue()); assertEquals("Unexpected number of "+type.getSimpleName()+"s", expectedNumberOfObjects, count.getValue()); } @@ -2162,12 +2159,9 @@ private void dumpOrg(StringBuilder sb, PrismObject org, int indent) { protected void displayUsers() throws SchemaException, ObjectNotFoundException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException { Task task = taskManager.createTaskInstance(AbstractModelIntegrationTest.class+".displayUsers"); OperationResult result = task.getResult(); - ResultHandler handler = new ResultHandler() { - @Override - public boolean handle(PrismObject user, OperationResult parentResult) { - display("User", user); - return true; - } + ResultHandler handler = (user, parentResult) -> { + display("User", user); + return true; }; modelService.searchObjectsIterative(UserType.class, null, handler, null, task, result); result.computeStatus(); @@ -4184,12 +4178,9 @@ protected void assertSecurityContextNoAuthorizationActions() { protected void displayAllUsers() throws SchemaException, ObjectNotFoundException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { Task task = taskManager.createTaskInstance(AbstractModelIntegrationTest.class.getName() + ".displayAllUsers"); OperationResult result = task.getResult(); - ResultHandler handler = new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult parentResult) { - display("User", object); - return true; - } + ResultHandler handler = (object, parentResult) -> { + display("User", object); + return true; }; modelService.searchObjectsIterative(UserType.class, null, handler, null, task, result); result.computeStatus(); diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ProvisioningServiceImpl.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ProvisioningServiceImpl.java index 58e3357a46e..687fc590f4c 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ProvisioningServiceImpl.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ProvisioningServiceImpl.java @@ -1000,7 +1000,7 @@ public SearchResultMetadata searchObjectsIterative(final try { - metadata = getCacheRepositoryService().searchObjectsIterative(type, query, internalHandler, repoOptions, false, result); // TODO think about strictSequential flag + metadata = getCacheRepositoryService().searchObjectsIterative(type, query, internalHandler, repoOptions, true, result); result.computeStatus(); result.recordSuccessIfUnknown(); diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java index 95fc8f772f1..ceae5ce7c6a 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/ShadowManager.java @@ -711,7 +711,7 @@ public SearchResultMetadata searchObjectsIterativeRepository( ObjectQuery repoQuery = query.clone(); processQueryMatchingRules(repoQuery, ctx.getObjectClassDefinition()); - return repositoryService.searchObjectsIterative(ShadowType.class, repoQuery, repoHandler, options, false, parentResult); // TODO think about strictSequential flag + return repositoryService.searchObjectsIterative(ShadowType.class, repoQuery, repoHandler, options, true, parentResult); } /** diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectAlreadyExistHandler.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectAlreadyExistHandler.java index 1d396fbd24e..c1cdf7dcfba 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectAlreadyExistHandler.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/errorhandling/ObjectAlreadyExistHandler.java @@ -218,11 +218,10 @@ private ObjectQuery createQueryBySecondaryIdentifier(ShadowType shadow) throws S * Note: this may return dead shadow. */ private List> findConflictingShadowsInRepo(ObjectQuery query, Task task, OperationResult parentResult) - throws ObjectNotFoundException, CommunicationException, ConfigurationException, SchemaException, - SecurityViolationException, ExpressionEvaluationException { + throws SchemaException { final List> foundAccount = new ArrayList<>(); - repositoryService.searchObjectsIterative(ShadowType.class, query, (object,result) -> foundAccount.add(object), null, false, parentResult); + repositoryService.searchObjectsIterative(ShadowType.class, query, (object,result) -> foundAccount.add(object), null, true, parentResult); return foundAccount; } diff --git a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationResultHandler.java b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationResultHandler.java index 8591caa275b..35e270dd93a 100644 --- a/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationResultHandler.java +++ b/provisioning/provisioning-impl/src/main/java/com/evolveum/midpoint/provisioning/impl/task/MultiPropagationResultHandler.java @@ -22,7 +22,6 @@ import com.evolveum.midpoint.prism.query.builder.QueryBuilder; import com.evolveum.midpoint.provisioning.impl.ShadowCache; import com.evolveum.midpoint.provisioning.ucf.api.GenericFrameworkException; -import com.evolveum.midpoint.repo.api.PreconditionViolationException; import com.evolveum.midpoint.repo.api.RepositoryService; import com.evolveum.midpoint.repo.common.task.AbstractSearchIterativeResultHandler; import com.evolveum.midpoint.schema.ResultHandler; @@ -55,7 +54,7 @@ public MultiPropagationResultHandler(Task coordinatorTask, String taskOperationP @Override protected boolean handleObject(PrismObject resource, Task workerTask, OperationResult taskResult) - throws CommonException, PreconditionViolationException { + throws CommonException { LOGGER.trace("Propagating provisioning operations on {}", resource); ObjectQuery query = new ObjectQuery(); @@ -72,7 +71,7 @@ protected boolean handleObject(PrismObject resource, Task workerTa return true; }; - repositoryService.searchObjectsIterative(ShadowType.class, query, handler, null, false, taskResult); + repositoryService.searchObjectsIterative(ShadowType.class, query, handler, null, true, taskResult); LOGGER.trace("Propagation of {} done", resource); diff --git a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/dummy/TestDummy.java b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/dummy/TestDummy.java index a610df19a07..467f792fdfc 100644 --- a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/dummy/TestDummy.java +++ b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/dummy/TestDummy.java @@ -2361,7 +2361,7 @@ public boolean handle(PrismObject shadow, OperationResult parentResu displayWhen(TEST_NAME); SearchResultMetadata searchMetadata; if (useRepo) { - searchMetadata = repositoryService.searchObjectsIterative(ShadowType.class, query, handler, null, false, result); + searchMetadata = repositoryService.searchObjectsIterative(ShadowType.class, query, handler, null, true, result); } else { searchMetadata = provisioningService.searchObjectsIterative(ShadowType.class, query, options, handler, null, result); } diff --git a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/opendj/TestOpenDj.java b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/opendj/TestOpenDj.java index 79cc13464df..fdec77515ee 100644 --- a/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/opendj/TestOpenDj.java +++ b/provisioning/provisioning-impl/src/test/java/com/evolveum/midpoint/provisioning/impl/opendj/TestOpenDj.java @@ -1418,14 +1418,11 @@ protected void assertShadows(int expectedCount) throws SchemaException { OperationResult result = new OperationResult(TestOpenDj.class.getName() + ".assertShadows"); int actualCount = repositoryService.countObjects(ShadowType.class, null, null, result); if (actualCount != expectedCount) { - ResultHandler handler = new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult parentResult) { - display("Repo shadow", object); - return true; - } + ResultHandler handler = (object, parentResult) -> { + display("Repo shadow", object); + return true; }; - repositoryService.searchObjectsIterative(ShadowType.class, null, handler, null, false, result); + repositoryService.searchObjectsIterative(ShadowType.class, null, handler, null, true, result); assertEquals("Unexpected number of shadows in the repo", expectedCount, actualCount); } } diff --git a/repo/repo-api/src/main/java/com/evolveum/midpoint/repo/api/RepositoryService.java b/repo/repo-api/src/main/java/com/evolveum/midpoint/repo/api/RepositoryService.java index 7606f7d4b5b..dac24882d71 100644 --- a/repo/repo-api/src/main/java/com/evolveum/midpoint/repo/api/RepositoryService.java +++ b/repo/repo-api/src/main/java/com/evolveum/midpoint/repo/api/RepositoryService.java @@ -297,17 +297,41 @@ SearchResultList searchContainers(Class type, Ob * @param handler * result handler * @param strictlySequential - * takes care not to skip any object nor to process objects more than once; - * currently requires paging NOT to be used - uses its own paging + * takes care not to skip any object nor to process objects more than once; see below * @param parentResult * parent OperationResult (in/out) - * @return all objects of specified type that match search criteria (subject - * to paging) + * @return all objects of specified type that match search criteria (subject to paging) * * @throws IllegalArgumentException * wrong object type * @throws SchemaException * unknown property used in search query + * + * A note related to iteration method: + * + * There are three iteration methods (see IterationMethodType): + * - SINGLE_TRANSACTION: Fetches objects in single DB transaction. Not supported for all DBMSs. + * - SIMPLE_PAGING: Uses the "simple paging" method: takes objects (e.g.) numbered 0 to 49, then 50 to 99, + * then 100 to 149, and so on. The disadvantage is that if the order of objects is changed + * during operation (e.g. by inserting/deleting some of them) then some objects can be + * processed multiple times, where others can be skipped. + * - STRICTLY_SEQUENTIAL_PAGING: Uses the "strictly sequential paging" method: sorting returned objects by OID. This + * is (almost) reliable in such a way that no object would be skipped. However, custom + * paging cannot be used in this mode. + * + * If GetOperationOptions.iterationMethod is specified, it is used without any further considerations. + * Otherwise, the repository configuration determines whether to use SINGLE_TRANSACTION or a paging. In the latter case, + * strictlySequential flag determines between SIMPLE_PAGING (if false) and STRICTLY_SEQUENTIAL_PAGING (if true). + * + * If explicit GetOperationOptions.iterationMethod is not provided, and paging is prescribed, and strictlySequential flag + * is true and client-provided paging conflicts with the paging used by the iteration method, a warning is issued, and + * iteration method is switched to SIMPLE_PAGING. + * + * Sources of conflicts: + * - ordering is specified + * - offset is specified + * (limit is not a problem) + * */ SearchResultMetadata searchObjectsIterative(Class type, ObjectQuery query, diff --git a/repo/repo-cache/src/main/java/com/evolveum/midpoint/repo/cache/RepositoryCache.java b/repo/repo-cache/src/main/java/com/evolveum/midpoint/repo/cache/RepositoryCache.java index 0573b0bfce9..0c1bb3e760a 100644 --- a/repo/repo-cache/src/main/java/com/evolveum/midpoint/repo/cache/RepositoryCache.java +++ b/repo/repo-cache/src/main/java/com/evolveum/midpoint/repo/cache/RepositoryCache.java @@ -352,12 +352,9 @@ public SearchResultMetadata searchObjectsIterative(Class< // TODO use cached query result if applicable log("Cache: PASS searchObjectsIterative ({})", type.getSimpleName()); final Cache cache = getCache(); - ResultHandler myHandler = new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult parentResult) { - cacheObject(cache, object, GetOperationOptions.isReadOnly(SelectorOptions.findRootOptions(options))); - return handler.handle(object, parentResult); - } + ResultHandler myHandler = (object, parentResult1) -> { + cacheObject(cache, object, GetOperationOptions.isReadOnly(SelectorOptions.findRootOptions(options))); + return handler.handle(object, parentResult1); }; Long startTime = repoOpStart(); try { diff --git a/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/RepoObjectResolver.java b/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/RepoObjectResolver.java index b54dc9542a1..bcca6b3428f 100644 --- a/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/RepoObjectResolver.java +++ b/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/RepoObjectResolver.java @@ -15,29 +15,22 @@ */ package com.evolveum.midpoint.repo.common; -import java.util.Collection; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; - import com.evolveum.midpoint.prism.PrismContext; -import com.evolveum.midpoint.prism.PrismObject; import com.evolveum.midpoint.prism.query.ObjectQuery; import com.evolveum.midpoint.repo.api.RepositoryService; import com.evolveum.midpoint.schema.GetOperationOptions; import com.evolveum.midpoint.schema.ResultHandler; import com.evolveum.midpoint.schema.SelectorOptions; import com.evolveum.midpoint.schema.result.OperationResult; -import com.evolveum.midpoint.schema.util.SimpleObjectResolver; import com.evolveum.midpoint.task.api.Task; -import com.evolveum.midpoint.util.exception.CommunicationException; -import com.evolveum.midpoint.util.exception.ConfigurationException; -import com.evolveum.midpoint.util.exception.ExpressionEvaluationException; import com.evolveum.midpoint.util.exception.ObjectNotFoundException; import com.evolveum.midpoint.util.exception.SchemaException; -import com.evolveum.midpoint.util.exception.SecurityViolationException; import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; + +import java.util.Collection; /** * This is only used in tests. But due to complicated dependencies this is @@ -60,17 +53,14 @@ public class RepoObjectResolver implements ObjectResolver { public void searchIterative(Class type, ObjectQuery query, Collection> options, ResultHandler handler, Task task, OperationResult parentResult) - throws SchemaException, ObjectNotFoundException, CommunicationException, ConfigurationException, - SecurityViolationException, ExpressionEvaluationException { - - cacheRepositoryService.searchObjectsIterative(type, query, handler, options, false, parentResult); + throws SchemaException { + cacheRepositoryService.searchObjectsIterative(type, query, handler, options, true, parentResult); } @Override public O resolve(ObjectReferenceType ref, Class expectedType, Collection> options, String contextDescription, Task task, - OperationResult result) throws ObjectNotFoundException, SchemaException, CommunicationException, - ConfigurationException, SecurityViolationException, ExpressionEvaluationException { + OperationResult result) { // TODO Auto-generated method stub return null; } @@ -78,8 +68,7 @@ public O resolve(ObjectReferenceType ref, Class expect @Override public O getObject(Class expectedType, String oid, Collection> options, Task task, OperationResult parentResult) - throws ObjectNotFoundException, CommunicationException, SchemaException, ConfigurationException, - SecurityViolationException, ExpressionEvaluationException { + throws ObjectNotFoundException, SchemaException { return cacheRepositoryService.getObject(expectedType, oid, options, parentResult).asObjectable(); } diff --git a/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/task/AbstractSearchIterativeTaskHandler.java b/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/task/AbstractSearchIterativeTaskHandler.java index 1fb8be9f98c..5e93486f7d2 100644 --- a/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/task/AbstractSearchIterativeTaskHandler.java +++ b/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/task/AbstractSearchIterativeTaskHandler.java @@ -241,7 +241,7 @@ public TaskWorkBucketProcessingResult run(Task localCoordinatorTask, WorkBucketT if (!useRepository) { searchIterative((Class) type, query, searchOptions, resultHandler, localCoordinatorTask, opResult); } else { - repositoryService.searchObjectsIterative((Class) type, query, resultHandler, searchOptions, false, opResult); // TODO think about this + repositoryService.searchObjectsIterative((Class) type, query, resultHandler, searchOptions, true, opResult); } resultHandler.completeProcessing(localCoordinatorTask, opResult); @@ -447,7 +447,7 @@ protected Integer countObjects(Class type, ObjectQuery */ protected void searchIterative(Class type, ObjectQuery query, Collection> searchOptions, ResultHandler resultHandler, Task coordinatorTask, OperationResult opResult) throws SchemaException, ObjectNotFoundException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { - repositoryService.searchObjectsIterative(type, query, resultHandler, searchOptions, false, opResult); // TODO think about this + repositoryService.searchObjectsIterative(type, query, resultHandler, searchOptions, true, opResult); } /** diff --git a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/AddGetObjectTest.java b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/AddGetObjectTest.java index d0a1980c8a0..fd29ae88e18 100644 --- a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/AddGetObjectTest.java +++ b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/AddGetObjectTest.java @@ -523,7 +523,7 @@ public boolean handle(PrismObject object, OperationResult parentResu } }; - repositoryService.searchObjectsIterative(ObjectType.class, null, handler, null, false, result); + repositoryService.searchObjectsIterative(ObjectType.class, null, handler, null, true, result); assertTrue(!objects.isEmpty()); } diff --git a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/ConcurrencyTest.java b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/ConcurrencyTest.java index aab5e1fc389..68fcfd0ebe3 100644 --- a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/ConcurrencyTest.java +++ b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/ConcurrencyTest.java @@ -488,7 +488,7 @@ public void test010SearchIterative() throws Exception { } return true; }, - null, false, result); + null, true, result); PrismObject reloaded = repositoryService.getObject(UserType.class, oid, null, result); AssertJUnit.assertEquals("Full name was not changed", newFullName, reloaded.asObjectable().getFullName().getOrig()); diff --git a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/SearchIterativeTest.java b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/SearchIterativeTest.java index 699aa06a483..240d949e8d7 100644 --- a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/SearchIterativeTest.java +++ b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/SearchIterativeTest.java @@ -18,6 +18,8 @@ import com.evolveum.midpoint.prism.PrismObject; import com.evolveum.midpoint.prism.polystring.PolyString; +import com.evolveum.midpoint.prism.query.ObjectPaging; +import com.evolveum.midpoint.prism.query.ObjectQuery; import com.evolveum.midpoint.prism.util.PrismTestUtil; import com.evolveum.midpoint.schema.MidPointPrismContextFactory; import com.evolveum.midpoint.schema.ResultHandler; @@ -49,6 +51,7 @@ @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) public class SearchIterativeTest extends BaseSQLRepoTest { + @SuppressWarnings("unused") private static final Trace LOGGER = TraceManager.getTrace(SearchIterativeTest.class); private static final int BASE = 100000; private static final int COUNT = 500; @@ -62,7 +65,7 @@ public void beforeClass() throws Exception { createObjects(); } - protected void createObjects() throws com.evolveum.midpoint.util.exception.ObjectAlreadyExistsException, com.evolveum.midpoint.util.exception.SchemaException { + private void createObjects() throws com.evolveum.midpoint.util.exception.ObjectAlreadyExistsException, com.evolveum.midpoint.util.exception.SchemaException { OperationResult result = new OperationResult("add objects"); for (int i = BASE; i < BASE + COUNT; i++) { UserType user = prismContext.getSchemaRegistry().findObjectDefinitionByCompileTimeClass(UserType.class).instantiate().asObjectable(); @@ -77,17 +80,14 @@ protected void createObjects() throws com.evolveum.midpoint.util.exception.Objec } @Test - public void test100SimpleIteration() throws Exception { - OperationResult result = new OperationResult("test100SimpleIteration"); + public void test100SimpleSequentialIteration() throws Exception { + OperationResult result = new OperationResult("test100SimpleSequentialIteration"); final List> objects = new ArrayList<>(); - ResultHandler handler = new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult parentResult) { - objects.add(object); - return true; - } + ResultHandler handler = (object, parentResult) -> { + objects.add(object); + return true; }; repositoryService.searchObjectsIterative(UserType.class, null, handler, null, true, result); @@ -98,17 +98,33 @@ public boolean handle(PrismObject object, OperationResult parentResult) { } @Test - public void test105SimpleIterationNoSequence() throws Exception { - OperationResult result = new OperationResult("test105SimpleIterationNoSequence"); + public void test102SimpleSequentialIterationWithMaxSize() throws Exception { + OperationResult result = new OperationResult("test102SimpleSequentialIterationWithMaxSize"); final List> objects = new ArrayList<>(); - ResultHandler handler = new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult parentResult) { - objects.add(object); - return true; - } + ResultHandler handler = (object, parentResult) -> { + objects.add(object); + return true; + }; + + ObjectQuery query = ObjectQuery.createObjectQuery(ObjectPaging.createPaging(null, 70)); + repositoryService.searchObjectsIterative(UserType.class, query, handler, null, true, result); + result.recomputeStatus(); + + assertTrue(result.isSuccess()); + assertObjects(objects, 70); + } + + @Test + public void test105SimpleNonSequentialIteration() throws Exception { + OperationResult result = new OperationResult("test105SimpleNonSequentialIteration"); + + final List> objects = new ArrayList<>(); + + ResultHandler handler = (object, parentResult) -> { + objects.add(object); + return true; }; repositoryService.searchObjectsIterative(UserType.class, null, handler, null, false, result); @@ -143,17 +159,14 @@ public void test110DeleteAll() throws Exception { final List> objects = new ArrayList<>(); - ResultHandler handler = new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult parentResult) { - objects.add(object); - try { - repositoryService.deleteObject(UserType.class, object.getOid(), parentResult); - } catch (ObjectNotFoundException e) { - throw new SystemException(e); - } - return true; + ResultHandler handler = (object, parentResult) -> { + objects.add(object); + try { + repositoryService.deleteObject(UserType.class, object.getOid(), parentResult); + } catch (ObjectNotFoundException e) { + throw new SystemException(e); } + return true; }; repositoryService.searchObjectsIterative(UserType.class, null, handler, null, true, result); @@ -174,20 +187,17 @@ public void test120DeleteHalf() throws Exception { final List> objects = new ArrayList<>(); - ResultHandler handler = new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult parentResult) { - objects.add(object); - try { - int number = Integer.parseInt(((UserType) object.asObjectable()).getCostCenter()); - if (number % 2 == 0) { - repositoryService.deleteObject(UserType.class, object.getOid(), parentResult); - } - } catch (ObjectNotFoundException e) { - throw new SystemException(e); + ResultHandler handler = (object, parentResult) -> { + objects.add(object); + try { + int number = Integer.parseInt(object.asObjectable().getCostCenter()); + if (number % 2 == 0) { + repositoryService.deleteObject(UserType.class, object.getOid(), parentResult); } - return true; + } catch (ObjectNotFoundException e) { + throw new SystemException(e); } + return true; }; repositoryService.searchObjectsIterative(UserType.class, null, handler, null, true, result); @@ -206,27 +216,24 @@ public void test130AddOneForOne() throws Exception { final List> objects = new ArrayList<>(); - ResultHandler handler = new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult parentResult) { - objects.add(object); - System.out.print("Got object " + object.getOid()); - try { - int number = Integer.parseInt(((UserType) object.asObjectable()).getCostCenter()); - if (number >= 0) { - UserType user = prismContext.getSchemaRegistry().findObjectDefinitionByCompileTimeClass(UserType.class).instantiate().asObjectable(); - user.setOid("user-" + (2*BASE + COUNT - number) + "-FF"); - user.setName(new PolyStringType(new PolyString("user-new-" + number))); - user.setCostCenter(String.valueOf(-number)); - repositoryService.addObject(user.asPrismObject(), null, parentResult); - System.out.print(" ... creating object " + user.getOid()); - } - } catch (ObjectAlreadyExistsException|SchemaException e) { - throw new SystemException(e); + ResultHandler handler = (object, parentResult) -> { + objects.add(object); + System.out.print("Got object " + object.getOid()); + try { + int number = Integer.parseInt(object.asObjectable().getCostCenter()); + if (number >= 0) { + UserType user = prismContext.getSchemaRegistry().findObjectDefinitionByCompileTimeClass(UserType.class).instantiate().asObjectable(); + user.setOid("user-" + (2*BASE + COUNT - number) + "-FF"); + user.setName(new PolyStringType(new PolyString("user-new-" + number))); + user.setCostCenter(String.valueOf(-number)); + repositoryService.addObject(user.asPrismObject(), null, parentResult); + System.out.print(" ... creating object " + user.getOid()); } - System.out.println(); - return true; + } catch (ObjectAlreadyExistsException|SchemaException e) { + throw new SystemException(e); } + System.out.println(); + return true; }; repositoryService.searchObjectsIterative(UserType.class, null, handler, null, true, result); diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/ObjectPagingAfterOid.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/ObjectPagingAfterOid.java index 8e9f3c79521..bb9f920c274 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/ObjectPagingAfterOid.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/ObjectPagingAfterOid.java @@ -20,7 +20,9 @@ /** * @author Pavol - */ // Temporary hack. Represents special paging object that means + */ + +// Temporary hack. Represents special paging object that means // "give me objects with OID greater than specified one, sorted by OID ascending". // // TODO: replace by using cookie that is part of the standard ObjectPaging diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryConfiguration.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryConfiguration.java index 450b49ec1c4..ddddc9f0791 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryConfiguration.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryConfiguration.java @@ -557,15 +557,8 @@ private void computeDefaultConcurrencyParameters() { } private void computeDefaultIterativeSearchParameters() { - if (isUsingH2()) { - defaultIterativeSearchByPaging = true; - defaultIterativeSearchByPagingBatchSize = 50; - } else if (isUsingMySqlCompatible()) { - defaultIterativeSearchByPaging = true; - defaultIterativeSearchByPagingBatchSize = 50; - } else { - defaultIterativeSearchByPaging = false; - } + defaultIterativeSearchByPaging = true; + defaultIterativeSearchByPagingBatchSize = 50; } /** diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java index 71e5718c247..36c83cdaf80 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/SqlRepositoryServiceImpl.java @@ -16,8 +16,6 @@ package com.evolveum.midpoint.repo.sql; -import com.evolveum.midpoint.common.LoggingConfigurationManager; -import com.evolveum.midpoint.common.ProfilingConfigurationManager; import com.evolveum.midpoint.common.configuration.api.MidpointConfiguration; import com.evolveum.midpoint.common.crypto.CryptoUtil; import com.evolveum.midpoint.prism.ConsistencyCheckScope; @@ -52,7 +50,6 @@ import com.evolveum.midpoint.schema.util.FocusTypeUtil; import com.evolveum.midpoint.schema.util.ObjectQueryUtil; import com.evolveum.midpoint.schema.util.ObjectTypeUtil; -import com.evolveum.midpoint.schema.util.SystemConfigurationTypeUtil; import com.evolveum.midpoint.util.DebugUtil; import com.evolveum.midpoint.util.PrettyPrinter; import com.evolveum.midpoint.util.QNameUtil; @@ -63,15 +60,12 @@ import com.evolveum.midpoint.util.exception.ObjectNotFoundException; import com.evolveum.midpoint.util.exception.SchemaException; import com.evolveum.midpoint.util.exception.SecurityViolationException; -import com.evolveum.midpoint.util.exception.SystemException; import com.evolveum.midpoint.util.logging.LoggingUtils; import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; import com.evolveum.midpoint.xml.ns._public.common.common_3.*; import com.evolveum.prism.xml.ns._public.query_3.SearchFilterType; -import com.evolveum.prism.xml.ns._public.types_3.PolyStringNormalizerConfigurationType; import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; -import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.Validate; import org.hibernate.Session; import org.hibernate.SessionFactory; @@ -377,12 +371,12 @@ private void logSearchInputParameters(Class type, ObjectQuery query, bool return; } - LOGGER.trace("Full query\n{}\nFull paging\n{}", new Object[]{ - (query == null ? "undefined" : query.debugDump()), - (paging != null ? paging.debugDump() : "undefined")}); + LOGGER.trace("Full query\n{}\nFull paging\n{}", + query == null ? "undefined" : query.debugDump(), + paging != null ? paging.debugDump() : "undefined"); if (iterative) { - LOGGER.trace("Iterative search by paging: {}, batch size {}", + LOGGER.trace("Iterative search by paging defined by the configuration: {}, batch size {}", getConfiguration().isIterativeSearchByPaging(), getConfiguration().getIterativeSearchByPagingBatchSize()); } @@ -825,20 +819,32 @@ public SearchResultMetadata searchObjectsIterative(Class< // against DB that does not support it, or if he requests simple paging where strictly sequential one is // indicated, we will obey (with a warning in some cases). IterationMethodType iterationMethod; - IterationMethodType specificIterationMethod = GetOperationOptions.getIterationMethod(SelectorOptions.findRootOptions(options)); - if (specificIterationMethod == null || specificIterationMethod == IterationMethodType.DEFAULT) { + IterationMethodType explicitIterationMethod = GetOperationOptions.getIterationMethod(SelectorOptions.findRootOptions(options)); + if (explicitIterationMethod == null || explicitIterationMethod == IterationMethodType.DEFAULT) { if (getConfiguration().isIterativeSearchByPaging()) { - iterationMethod = strictlySequential ? IterationMethodType.STRICTLY_SEQUENTIAL_PAGING : IterationMethodType.SIMPLE_PAGING; + if (strictlySequential) { + if (isCustomPagingOkWithPagedSeqIteration(query)) { + iterationMethod = IterationMethodType.STRICTLY_SEQUENTIAL_PAGING; + } else { + // TODO switch to LOGGER.error + throw new IllegalArgumentException("Iterative search was defined in the repository configuration, and strict sequentiality " + + "was requested. However, a custom paging precludes its application. Therefore switching to " + + "simple paging iteration method. Paging requested: " + query.getPaging()); + //iterationMethod = IterationMethodType.SIMPLE_PAGING; + } + } else { + iterationMethod = IterationMethodType.SIMPLE_PAGING; + } } else { iterationMethod = IterationMethodType.SINGLE_TRANSACTION; } } else { - iterationMethod = specificIterationMethod; + iterationMethod = explicitIterationMethod; } if (strictlySequential && iterationMethod == IterationMethodType.SIMPLE_PAGING) { LOGGER.warn("Using simple paging where strictly sequential one is indicated: type={}, query={}", type, query); - } else if (getConfiguration().isIterativeSearchByPaging() && specificIterationMethod == IterationMethodType.SINGLE_TRANSACTION) { + } else if (getConfiguration().isIterativeSearchByPaging() && explicitIterationMethod == IterationMethodType.SINGLE_TRANSACTION) { // we should introduce some 'native iteration supported' flag for the DB configuration to avoid false warnings here // based on 'iterativeSearchByPaging' setting for databases that support native iteration LOGGER.warn("Using single transaction iteration where DB indicates paging should be used: type={}, query={}", type, query); @@ -855,6 +861,14 @@ public SearchResultMetadata searchObjectsIterative(Class< return null; } + public static boolean isCustomPagingOkWithPagedSeqIteration(ObjectQuery query) { + if (query == null || query.getPaging() == null) { + return true; + } + ObjectPaging paging = query.getPaging(); + return !paging.hasOrdering() && !paging.hasGrouping() && paging.getOffset() == null; + } + @Nullable private SearchResultMetadata searchObjectsIterativeBySingleTransaction(Class type, ObjectQuery query, ResultHandler handler, Collection> options, OperationResult subResult) diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/ObjectRetriever.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/ObjectRetriever.java index efa82c18677..41271460bc0 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/ObjectRetriever.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/ObjectRetriever.java @@ -65,6 +65,7 @@ import java.util.*; import static org.apache.commons.lang3.ArrayUtils.getLength; +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; /** * @author lazyman, mederly @@ -851,7 +852,7 @@ public void searchObjectsIterativeByPaging(Class type, * * Constraints: * - There can be no ordering prescribed. We use our own ordering. - * - Moreover, for simplicity we disallow any explicit paging. + * - We also disallow any explicit paging - except for maxSize setting. * * Implementation is very simple - we fetch objects ordered by OID, and remember last OID fetched. * Obviously no object will be present in output more than once. @@ -863,20 +864,28 @@ public void searchObjectsIterativeByPagingStrictlySequent throws SchemaException { try { - ObjectQuery pagedQuery = query != null ? query.clone() : new ObjectQuery(); + if (!SqlRepositoryServiceImpl.isCustomPagingOkWithPagedSeqIteration(query)) { + throw new IllegalArgumentException("Externally specified paging is not supported on strictly sequential " + + "iterative search. Query = " + query); + } + Integer maxSize; + ObjectQuery pagedQuery; + if (query != null) { + maxSize = query.getPaging() != null ? query.getPaging().getMaxSize() : null; + pagedQuery = query.clone(); + } else { + maxSize = null; + pagedQuery = new ObjectQuery(); + } String lastOid = ""; final int batchSize = getConfiguration().getIterativeSearchByPagingBatchSize(); - if (pagedQuery.getPaging() != null) { - throw new IllegalArgumentException("Externally specified paging is not supported on strictly sequential iterative search."); - } - ObjectPagingAfterOid paging = new ObjectPagingAfterOid(); pagedQuery.setPaging(paging); main: for (;;) { paging.setOidGreaterThan(lastOid); - paging.setMaxSize(batchSize); + paging.setMaxSize(Math.min(batchSize, defaultIfNull(maxSize, Integer.MAX_VALUE))); List> objects = repositoryService.searchObjects(type, pagedQuery, options, result); @@ -886,10 +895,15 @@ public void searchObjectsIterativeByPagingStrictlySequent break main; } } - - if (objects.size() == 0) { + if (objects.size() == 0) { break; } + if (maxSize != null) { + maxSize -= objects.size(); + if (maxSize <= 0) { + break; + } + } } } finally { if (result.isUnknown()) { @@ -898,7 +912,7 @@ public void searchObjectsIterativeByPagingStrictlySequent } } - public boolean isAnySubordinateAttempt(String upperOrgOid, Collection lowerObjectOids) { + public boolean isAnySubordinateAttempt(String upperOrgOid, Collection lowerObjectOids) { Session session = null; try { session = baseHelper.beginReadOnlyTransaction(); diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/OrgClosureManager.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/OrgClosureManager.java index 8f0a27ba2bc..06c1c7e4036 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/OrgClosureManager.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/OrgClosureManager.java @@ -376,7 +376,7 @@ public boolean handle(PrismObject object, OperationResult parentResult) return true; } }; - repositoryService.searchObjectsIterative(OrgType.class, new ObjectQuery(), handler, null, false, result); + repositoryService.searchObjectsIterative(OrgType.class, new ObjectQuery(), handler, null, true, result); LOGGER.info("Org closure table was successfully recomputed (not committed yet); all {} organizations processed", orgsTotal); diff --git a/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java b/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java index ffa3c7737cd..11188a8b9d3 100644 --- a/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java +++ b/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/AbstractIntegrationTest.java @@ -1233,14 +1233,11 @@ protected void assertShadows(int expected, OperationResult result) throws Schema if (actual > 20) { AssertJUnit.fail("Unexpected number of (repository) shadows. Expected " + expected + " but was " + actual + " (too many to display)"); } - ResultHandler handler = new ResultHandler() { - @Override - public boolean handle(PrismObject object, OperationResult parentResult) { - display("found shadow", object); - return true; - } + ResultHandler handler = (object, parentResult) -> { + display("found shadow", object); + return true; }; - repositoryService.searchObjectsIterative(ShadowType.class, null, handler, null, false, result); + repositoryService.searchObjectsIterative(ShadowType.class, null, handler, null, true, result); AssertJUnit.fail("Unexpected number of (repository) shadows. Expected " + expected + " but was " + actual); } } diff --git a/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/RepoSimpleObjectResolver.java b/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/RepoSimpleObjectResolver.java index 0078848dadf..47c70c56a75 100644 --- a/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/RepoSimpleObjectResolver.java +++ b/repo/repo-test-util/src/main/java/com/evolveum/midpoint/test/RepoSimpleObjectResolver.java @@ -15,11 +15,6 @@ */ package com.evolveum.midpoint.test; -import java.util.Collection; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; - import com.evolveum.midpoint.prism.PrismContext; import com.evolveum.midpoint.prism.PrismObject; import com.evolveum.midpoint.prism.query.ObjectQuery; @@ -29,13 +24,13 @@ import com.evolveum.midpoint.schema.SelectorOptions; import com.evolveum.midpoint.schema.result.OperationResult; import com.evolveum.midpoint.schema.util.SimpleObjectResolver; -import com.evolveum.midpoint.util.exception.CommunicationException; -import com.evolveum.midpoint.util.exception.ConfigurationException; -import com.evolveum.midpoint.util.exception.ExpressionEvaluationException; import com.evolveum.midpoint.util.exception.ObjectNotFoundException; import com.evolveum.midpoint.util.exception.SchemaException; -import com.evolveum.midpoint.util.exception.SecurityViolationException; import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; + +import java.util.Collection; /** * @author semancik @@ -61,11 +56,8 @@ public PrismObject getObject(Class expectedType, St public void searchIterative(Class type, ObjectQuery query, Collection> options, ResultHandler handler, Object task, OperationResult parentResult) - throws SchemaException, ObjectNotFoundException, CommunicationException, ConfigurationException, - SecurityViolationException, ExpressionEvaluationException { - - cacheRepositoryService.searchObjectsIterative(type, query, handler, options, false, parentResult); - + throws SchemaException { + cacheRepositoryService.searchObjectsIterative(type, query, handler, options, true, parentResult); } diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/DeleteRepositoryAction.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/DeleteRepositoryAction.java index bb42576a030..e96eb9539c8 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/DeleteRepositoryAction.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/DeleteRepositoryAction.java @@ -82,7 +82,7 @@ private void deleteByFilter(ObjectQuery query) throws SchemaException, IOExcepti } private void deleteByFilter(ObjectTypes type, ObjectQuery query, OperationStatus operation, OperationResult result) - throws SchemaException, IOException { + throws SchemaException { ResultHandler handler = (prismObject, operationResult) -> { @@ -119,7 +119,7 @@ private void deleteByFilter(ObjectTypes type, ObjectQuery query, OperationStatus } RepositoryService repository = context.getRepository(); - repository.searchObjectsIterative(type.getClassDefinition(), query, handler, opts, false, result); + repository.searchObjectsIterative(type.getClassDefinition(), query, handler, opts, true, result); } private State askForState(PrismObject object) throws IOException { diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/worker/ExportProducerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/worker/ExportProducerWorker.java index 3f53669e5e0..c31b2f3e46b 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/worker/ExportProducerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/worker/ExportProducerWorker.java @@ -75,7 +75,7 @@ public void run() { }; RepositoryService repository = context.getRepository(); - repository.searchObjectsIterative(type.getClassDefinition(), query, handler, opts, false, operation.getResult()); + repository.searchObjectsIterative(type.getClassDefinition(), query, handler, opts, true, operation.getResult()); } catch (SchemaException ex) { log.error("Unexpected exception, reason: {}", ex, ex.getMessage()); } catch (NinjaException ex) {