From bccb9d032dd8123b45b9611ebee6fd6875b13d26 Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Thu, 7 Feb 2019 13:19:02 +0100 Subject: [PATCH 1/6] Remove task only if in CLOSED state (MID-5033) It looks like that when re-scheduling already closed single-run tasks they keep their completionTimestamp set. And so they are eventually cleaned up. This is an immediate fix, requiring tasks that are to be cleaned to have also execution status = CLOSED. See also MID-5133. --- .../midpoint/task/quartzimpl/TaskManagerQuartzImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repo/task-quartz-impl/src/main/java/com/evolveum/midpoint/task/quartzimpl/TaskManagerQuartzImpl.java b/repo/task-quartz-impl/src/main/java/com/evolveum/midpoint/task/quartzimpl/TaskManagerQuartzImpl.java index 187b0a1a121..c17355f9422 100644 --- a/repo/task-quartz-impl/src/main/java/com/evolveum/midpoint/task/quartzimpl/TaskManagerQuartzImpl.java +++ b/repo/task-quartz-impl/src/main/java/com/evolveum/midpoint/task/quartzimpl/TaskManagerQuartzImpl.java @@ -2151,7 +2151,8 @@ public void cleanupTasks(CleanupPolicyType policy, Task executionTask, Operation List> obsoleteTasks; try { ObjectQuery obsoleteTasksQuery = prismContext.queryFor(TaskType.class) - .item(TaskType.F_COMPLETION_TIMESTAMP).le(timeXml) + .item(TaskType.F_EXECUTION_STATUS).eq(TaskExecutionStatusType.CLOSED) + .and().item(TaskType.F_COMPLETION_TIMESTAMP).le(timeXml) .and().item(TaskType.F_PARENT).isNull() .build(); obsoleteTasks = repositoryService.searchObjects(TaskType.class, obsoleteTasksQuery, null, result); From 7ce1f01a2bc11630233af54181722fc36d941e37 Mon Sep 17 00:00:00 2001 From: Radovan Semancik Date: Thu, 7 Feb 2019 16:03:05 +0100 Subject: [PATCH 2/6] Attempts to reproduce ConnId thread issues (MID-5099). Not successful. --- .../midpoint/schema/util/ObjectTypeUtil.java | 11 +- .../midpoint/test/ldap/OpenDJController.java | 3 +- .../test/AbstractModelIntegrationTest.java | 4 + .../resources/ldap-sync-massive/kraken.ldif | 10 + .../ldap-sync-massive/resource-opendj-bad.xml | 249 ++++++++++++++++++ .../ldap-sync-massive/resource-opendj.xml | 237 +++++++++++++++++ .../ldap-sync-massive/task-live-sync.xml | 44 ++++ .../resources/ldap-sync-massive/will.ldif | 15 ++ testing/story/testng-integration.xml | 1 + 9 files changed, 572 insertions(+), 2 deletions(-) create mode 100644 testing/story/src/test/resources/ldap-sync-massive/kraken.ldif create mode 100644 testing/story/src/test/resources/ldap-sync-massive/resource-opendj-bad.xml create mode 100644 testing/story/src/test/resources/ldap-sync-massive/resource-opendj.xml create mode 100644 testing/story/src/test/resources/ldap-sync-massive/task-live-sync.xml create mode 100644 testing/story/src/test/resources/ldap-sync-massive/will.ldif diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/ObjectTypeUtil.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/ObjectTypeUtil.java index 95d451d7819..d5157468a50 100644 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/ObjectTypeUtil.java +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/ObjectTypeUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2017 Evolveum + * Copyright (c) 2010-2019 Evolveum * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -583,6 +583,15 @@ public static T getExtensionItemRealValue(@Nullable ExtensionType extension, Item item = extension.asPrismContainerValue().findItem(ItemName.fromQName(itemName)); return item != null ? (T) item.getRealValue() : null; } + + public static T getExtensionItemRealValue(@NotNull PrismObject object, @NotNull QName itemName) { + PrismContainer extension = object.getExtension(); + if (extension == null) { + return null; + } + Item item = extension.findItem(ItemName.fromQName(itemName)); + return item != null ? (T) item.getRealValue() : null; + } public static void normalizeRelation(ObjectReferenceType reference, RelationRegistry relationRegistry) { if (reference != null) { diff --git a/infra/test-util/src/main/java/com/evolveum/midpoint/test/ldap/OpenDJController.java b/infra/test-util/src/main/java/com/evolveum/midpoint/test/ldap/OpenDJController.java index 719a6c5ea16..82c5db3d8ef 100755 --- a/infra/test-util/src/main/java/com/evolveum/midpoint/test/ldap/OpenDJController.java +++ b/infra/test-util/src/main/java/com/evolveum/midpoint/test/ldap/OpenDJController.java @@ -754,11 +754,12 @@ public void addEntry(Entry ldapEntry) { } } - public void addEntry(String ldif) throws IOException, LDIFException { + public Entry addEntry(String ldif) throws IOException, LDIFException { LDIFImportConfig importConfig = new LDIFImportConfig(IOUtils.toInputStream(ldif, "utf-8")); LDIFReader ldifReader = new LDIFReader(importConfig); Entry ldifEntry = ldifReader.readEntry(); addEntry(ldifEntry); + return ldifEntry; } public ChangeRecordEntry executeRenameChange(File file) throws LDIFException, IOException{ diff --git a/model/model-test/src/main/java/com/evolveum/midpoint/model/test/AbstractModelIntegrationTest.java b/model/model-test/src/main/java/com/evolveum/midpoint/model/test/AbstractModelIntegrationTest.java index bcaa6ecbd66..7126bed4a86 100644 --- a/model/model-test/src/main/java/com/evolveum/midpoint/model/test/AbstractModelIntegrationTest.java +++ b/model/model-test/src/main/java/com/evolveum/midpoint/model/test/AbstractModelIntegrationTest.java @@ -3134,6 +3134,10 @@ protected OperationResult waitForTaskNextRunAssertSuccess(Task origTask, final b return taskResult; } + protected OperationResult waitForTaskNextRun(final String taskOid) throws Exception { + return waitForTaskNextRun(taskOid, false, DEFAULT_TASK_WAIT_TIMEOUT, false); + } + protected OperationResult waitForTaskNextRun(final String taskOid, final boolean checkSubresult, final int timeout) throws Exception { return waitForTaskNextRun(taskOid, checkSubresult, timeout, false); } diff --git a/testing/story/src/test/resources/ldap-sync-massive/kraken.ldif b/testing/story/src/test/resources/ldap-sync-massive/kraken.ldif new file mode 100644 index 00000000000..27c3d35afcf --- /dev/null +++ b/testing/story/src/test/resources/ldap-sync-massive/kraken.ldif @@ -0,0 +1,10 @@ +dn: uid=kraken,ou=People,dc=example,dc=com +uid: kraken +cn: Kraken Krakenoff +sn: Kraken +givenname: Krakenoff +objectclass: top +objectclass: person +objectclass: organizationalPerson +objectclass: inetOrgPerson + diff --git a/testing/story/src/test/resources/ldap-sync-massive/resource-opendj-bad.xml b/testing/story/src/test/resources/ldap-sync-massive/resource-opendj-bad.xml new file mode 100644 index 00000000000..272fcab3233 --- /dev/null +++ b/testing/story/src/test/resources/ldap-sync-massive/resource-opendj-bad.xml @@ -0,0 +1,249 @@ + + + + + + + + Embedded Test OpenDJ + + + Dummy description, just for the test + + + c:connectorType + com.evolveum.polygon.connector.ldap.LdapConnector + + + + + + + + 10389 + localhost + dc=example,dc=com + cn=directory manager + secret + auto + entryUUID + ds-pwp-account-disabled + isMemberOf + + + + false + false + false + + + + 1 + 3 + + + + 1 + + + + + + + + account + default + Default Account + true + ri:inetOrgPerson + + ri:dn + Distinguished Name + + + + $user/name + + + + + + + + + ri:cn + Common Name + + + fullName + + + + weak + + fullName + + + + + + ri:sn + + + familyName + + + + weak + + familyName + + + + + + ri:givenName + + + givenName + + + + weak + + givenName + + + + + + ri:uid + + + weak + + $user/name + + + + weak + + name + + + + + + + + http://prism.evolveum.com/xml/ns/public/matching-rule-3#stringIgnoreCase + attributes/ri:dn + uid=idm,ou=Administrators,dc=example,dc=com + + + + + + + + + + + + + + + + + + + + + + ri:ds-pwp-account-disabled + + true + + + + + + + + true + ri:inetOrgPerson + + + + declare namespace c="http://midpoint.evolveum.com/xml/ns/public/common/common-3"; + c:name + + + + declare namespace c="http://midpoint.evolveum.com/xml/ns/public/common/common-3"; + declare namespace dj="http://midpoint.evolveum.com/xml/ns/public/resource/instance/ef2bc95b-76e0-59e2-86d6-3d4f02d3ffff"; + $c:account/c:attributes/dj:uid + + + + + + linked + true + + + deleted + true + + http://midpoint.evolveum.com/xml/ns/public/model/action-3#unlink + + + + unlinked + true + + http://midpoint.evolveum.com/xml/ns/public/model/action-3#link + + + + unmatched + true + + http://midpoint.evolveum.com/xml/ns/public/model/action-3#addFocus + + + + + + diff --git a/testing/story/src/test/resources/ldap-sync-massive/resource-opendj.xml b/testing/story/src/test/resources/ldap-sync-massive/resource-opendj.xml new file mode 100644 index 00000000000..b65e3d10058 --- /dev/null +++ b/testing/story/src/test/resources/ldap-sync-massive/resource-opendj.xml @@ -0,0 +1,237 @@ + + + + + + Embedded Test OpenDJ + + + Dummy description, just for the test + + + c:connectorType + com.evolveum.polygon.connector.ldap.LdapConnector + + + + + + + + 10389 + localhost + dc=example,dc=com + cn=directory manager + secret + auto + entryUUID + ds-pwp-account-disabled + isMemberOf + + + + false + false + false + + + + + + + + account + default + Default Account + true + ri:inetOrgPerson + + ri:dn + Distinguished Name + + + + $user/name + + + + + + + + + ri:cn + Common Name + + + fullName + + + + weak + + fullName + + + + + + ri:sn + + + familyName + + + + weak + + familyName + + + + + + ri:givenName + + + givenName + + + + weak + + givenName + + + + + + ri:uid + + + weak + + $user/name + + + + weak + + name + + + + + + + + http://prism.evolveum.com/xml/ns/public/matching-rule-3#stringIgnoreCase + attributes/ri:dn + uid=idm,ou=Administrators,dc=example,dc=com + + + + + + + + + + + + + + + + + + + + + + ri:ds-pwp-account-disabled + + true + + + + + + + + true + ri:inetOrgPerson + + + + declare namespace c="http://midpoint.evolveum.com/xml/ns/public/common/common-3"; + c:name + + + + declare namespace c="http://midpoint.evolveum.com/xml/ns/public/common/common-3"; + declare namespace dj="http://midpoint.evolveum.com/xml/ns/public/resource/instance/ef2bc95b-76e0-59e2-86d6-3d4f02d3ffff"; + $c:account/c:attributes/dj:uid + + + + + + linked + true + + + deleted + true + + http://midpoint.evolveum.com/xml/ns/public/model/action-3#unlink + + + + unlinked + true + + http://midpoint.evolveum.com/xml/ns/public/model/action-3#link + + + + unmatched + true + + http://midpoint.evolveum.com/xml/ns/public/model/action-3#addFocus + + + + + + diff --git a/testing/story/src/test/resources/ldap-sync-massive/task-live-sync.xml b/testing/story/src/test/resources/ldap-sync-massive/task-live-sync.xml new file mode 100644 index 00000000000..9b030a5ecaf --- /dev/null +++ b/testing/story/src/test/resources/ldap-sync-massive/task-live-sync.xml @@ -0,0 +1,44 @@ + + + + + + Live Sync: OpenDJ + + + ri:inetOrgPerson + + + eba4a816-2a05-11e9-9123-03a2334b9b4c + + runnable + + http://midpoint.evolveum.com/xml/ns/public/model/synchronization/task/live-sync/handler-3 + + recurring + tight + + 1 + + + diff --git a/testing/story/src/test/resources/ldap-sync-massive/will.ldif b/testing/story/src/test/resources/ldap-sync-massive/will.ldif new file mode 100644 index 00000000000..2f4de887c57 --- /dev/null +++ b/testing/story/src/test/resources/ldap-sync-massive/will.ldif @@ -0,0 +1,15 @@ +dn: uid=will,ou=People,dc=example,dc=com +uid: will +cn: Will Turner +sn: Turner +givenname: Will +objectclass: top +objectclass: person +objectclass: organizationalPerson +objectclass: inetOrgPerson +l: Caribbean +mail: will.turner@blackpearl.com +telephonenumber: +1 408 555 1234 +facsimiletelephonenumber: +1 408 555 4321 +userpassword: supersecret + diff --git a/testing/story/testng-integration.xml b/testing/story/testng-integration.xml index 57ca7dcee62..aba8c2298ea 100644 --- a/testing/story/testng-integration.xml +++ b/testing/story/testng-integration.xml @@ -58,6 +58,7 @@ + From cc1937857c0e31b4491d5ba536b35a43da6343b4 Mon Sep 17 00:00:00 2001 From: Radovan Semancik Date: Thu, 7 Feb 2019 17:49:57 +0100 Subject: [PATCH 3/6] Upgrade to ConnId framework 1.5.0.8 (MID-5099) --- build-system/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/pom.xml b/build-system/pom.xml index 54edadc62d8..d5086c418d9 100644 --- a/build-system/pom.xml +++ b/build-system/pom.xml @@ -83,7 +83,7 @@ 5.22.0 1.3 2.0.6 - 1.5.0.0 + 1.5.0.8 6.5.0 10.11.1.1 1.8.0 From 62a2058a6d4dbca5a05add3667342c409672ee55 Mon Sep 17 00:00:00 2001 From: Radovan Semancik Date: Thu, 7 Feb 2019 17:51:02 +0100 Subject: [PATCH 4/6] Fixed ConnId timeout configuration for update operations (MID-5126) --- .../ConnIdConfigurationTransformer.java | 9 ++- .../connid/ConnectorFactoryConnIdImpl.java | 30 +++++---- .../story/TestMisbehavingResources.java | 64 +++++++++++++++++-- .../story/src/test/resources/logback-test.xml | 1 + 4 files changed, 80 insertions(+), 24 deletions(-) diff --git a/provisioning/ucf-impl-connid/src/main/java/com/evolveum/midpoint/provisioning/ucf/impl/connid/ConnIdConfigurationTransformer.java b/provisioning/ucf-impl-connid/src/main/java/com/evolveum/midpoint/provisioning/ucf/impl/connid/ConnIdConfigurationTransformer.java index 2a893f511c5..85475957afb 100644 --- a/provisioning/ucf-impl-connid/src/main/java/com/evolveum/midpoint/provisioning/ucf/impl/connid/ConnIdConfigurationTransformer.java +++ b/provisioning/ucf-impl-connid/src/main/java/com/evolveum/midpoint/provisioning/ucf/impl/connid/ConnIdConfigurationTransformer.java @@ -18,6 +18,7 @@ import java.io.File; import java.lang.reflect.Array; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import javax.xml.namespace.QName; @@ -245,9 +246,11 @@ private void transformConnectorTimeoutsConfiguration(APIConfiguration apiConfig, if (SchemaConstants.NS_ICF_CONFIGURATION.equals(propertQName.getNamespaceURI())) { String opName = propertQName.getLocalPart(); - Class apiOpClass = ConnectorFactoryConnIdImpl.resolveApiOpClass(opName); - if (apiOpClass != null) { - apiConfig.setTimeout(apiOpClass, parseInt(prismProperty)); + Collection> apiOpClasses = ConnectorFactoryConnIdImpl.resolveApiOpClass(opName); + if (apiOpClasses != null) { + for (Class apiOpClass : apiOpClasses) { + apiConfig.setTimeout(apiOpClass, parseInt(prismProperty)); + } } else { throw new SchemaException("Unknown operation name " + opName + " in " + ConnectorFactoryConnIdImpl.CONNECTOR_SCHEMA_TIMEOUTS_XML_ELEMENT_NAME); diff --git a/provisioning/ucf-impl-connid/src/main/java/com/evolveum/midpoint/provisioning/ucf/impl/connid/ConnectorFactoryConnIdImpl.java b/provisioning/ucf-impl-connid/src/main/java/com/evolveum/midpoint/provisioning/ucf/impl/connid/ConnectorFactoryConnIdImpl.java index 1d1d94d9eef..04af09a8226 100644 --- a/provisioning/ucf-impl-connid/src/main/java/com/evolveum/midpoint/provisioning/ucf/impl/connid/ConnectorFactoryConnIdImpl.java +++ b/provisioning/ucf-impl-connid/src/main/java/com/evolveum/midpoint/provisioning/ucf/impl/connid/ConnectorFactoryConnIdImpl.java @@ -26,6 +26,7 @@ import java.net.URL; import java.security.Key; import java.util.Arrays; +import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; @@ -69,6 +70,7 @@ import org.identityconnectors.framework.api.operations.SyncApiOp; import org.identityconnectors.framework.api.operations.TestApiOp; import org.identityconnectors.framework.api.operations.UpdateApiOp; +import org.identityconnectors.framework.api.operations.UpdateDeltaApiOp; import org.identityconnectors.framework.api.operations.ValidateApiOp; import org.identityconnectors.framework.common.FrameworkUtil; import org.springframework.beans.factory.annotation.Autowired; @@ -153,7 +155,7 @@ public class ConnectorFactoryConnIdImpl implements ConnectorFactory { public static final String CONNECTOR_SCHEMA_RESULTS_HANDLER_CONFIGURATION_ENABLE_CASE_INSENSITIVE_HANDLER = "enableCaseInsensitiveFilter"; public static final String CONNECTOR_SCHEMA_RESULTS_HANDLER_CONFIGURATION_ENABLE_ATTRIBUTES_TO_GET_SEARCH_RESULTS_HANDLER = "enableAttributesToGetSearchResultsHandler"; - static final Map> apiOpMap = new HashMap<>(); + static final Map>> apiOpMap = new HashMap<>(); private static final String ICF_CONFIGURATION_NAMESPACE_PREFIX = SchemaConstants.ICF_FRAMEWORK_URI + "/bundle/"; private static final String CONNECTOR_IDENTIFIER_SEPARATOR = "/"; @@ -892,23 +894,23 @@ public void access(char[] decryptedChars) { result.computeStatus(); } - static Class resolveApiOpClass(String opName) { + static Collection> resolveApiOpClass(String opName) { return apiOpMap.get(opName); } static { - apiOpMap.put("create", CreateApiOp.class); - apiOpMap.put("get", GetApiOp.class); - apiOpMap.put("update", UpdateApiOp.class); - apiOpMap.put("delete", DeleteApiOp.class); - apiOpMap.put("test", TestApiOp.class); - apiOpMap.put("scriptOnConnector", ScriptOnConnectorApiOp.class); - apiOpMap.put("scriptOnResource", ScriptOnResourceApiOp.class); - apiOpMap.put("authentication", AuthenticationApiOp.class); - apiOpMap.put("search", SearchApiOp.class); - apiOpMap.put("validate", ValidateApiOp.class); - apiOpMap.put("sync", SyncApiOp.class); - apiOpMap.put("schema", SchemaApiOp.class); + apiOpMap.put("create", Arrays.asList(CreateApiOp.class)); + apiOpMap.put("get", Arrays.asList(GetApiOp.class)); + apiOpMap.put("update", Arrays.asList(UpdateApiOp.class, UpdateDeltaApiOp.class)); + apiOpMap.put("delete", Arrays.asList(DeleteApiOp.class)); + apiOpMap.put("test", Arrays.asList(TestApiOp.class)); + apiOpMap.put("scriptOnConnector", Arrays.asList(ScriptOnConnectorApiOp.class)); + apiOpMap.put("scriptOnResource", Arrays.asList(ScriptOnResourceApiOp.class)); + apiOpMap.put("authentication", Arrays.asList(AuthenticationApiOp.class)); + apiOpMap.put("search", Arrays.asList(SearchApiOp.class)); + apiOpMap.put("validate", Arrays.asList(ValidateApiOp.class)); + apiOpMap.put("sync", Arrays.asList(SyncApiOp.class)); + apiOpMap.put("schema", Arrays.asList(SchemaApiOp.class)); } @Override diff --git a/testing/story/src/test/java/com/evolveum/midpoint/testing/story/TestMisbehavingResources.java b/testing/story/src/test/java/com/evolveum/midpoint/testing/story/TestMisbehavingResources.java index f07bce6ce1f..6c8c5c2766c 100644 --- a/testing/story/src/test/java/com/evolveum/midpoint/testing/story/TestMisbehavingResources.java +++ b/testing/story/src/test/java/com/evolveum/midpoint/testing/story/TestMisbehavingResources.java @@ -33,6 +33,7 @@ import com.evolveum.midpoint.schema.result.OperationResult; import com.evolveum.midpoint.task.api.Task; import com.evolveum.midpoint.test.util.MidPointTestConstants; +import com.evolveum.midpoint.xml.ns._public.common.common_3.UserType; /** * Test for various resource-side errors, strange situations, timeouts @@ -48,6 +49,8 @@ public class TestMisbehavingResources extends AbstractStoryTest { protected static final File RESOURCE_DUMMY_FILE = new File(TEST_DIR, "resource-dummy.xml"); protected static final String RESOURCE_DUMMY_OID = "5f9615a2-d05b-11e8-9dab-37186a8ab7ef"; + + private static final String USER_JACK_FULL_NAME_CAPTAIN = "Captain Jack Sparrow"; @Override public void initSystem(Task initTask, OperationResult initResult) throws Exception { @@ -98,7 +101,7 @@ public void test019SanityUnassignJackDummyAccount() throws Exception { } /** - * MID-4773 + * MID-4773, MID-5099 */ @Test public void test100AssignJackDummyAccountTimeout() throws Exception { @@ -119,11 +122,7 @@ public void test100AssignJackDummyAccountTimeout() throws Exception { displayThen(TEST_NAME); assertInProgress(result); - // ConnId timeout is obviously not enforced. Therefore if the operation - // does not fail by itself it is not forcibly stopped. The account is - // created anyway. - assertDummyAccountByUsername(null, USER_JACK_USERNAME) - .assertFullName(USER_JACK_FULL_NAME); + assertNoDummyAccount(USER_JACK_USERNAME); } @Test @@ -140,7 +139,7 @@ public void test102AssignJackDummyAccounRetry() throws Exception { // WHEN displayWhen(TEST_NAME); - recomputeUser(USER_JACK_OID, task, result); + reconcileUser(USER_JACK_OID, task, result); // THEN displayThen(TEST_NAME); @@ -149,4 +148,55 @@ public void test102AssignJackDummyAccounRetry() throws Exception { assertDummyAccountByUsername(null, USER_JACK_USERNAME) .assertFullName(USER_JACK_FULL_NAME); } + + /** + * MID-5126 + */ + @Test + public void test110ModifyJackDummyAccountTimeout() throws Exception { + final String TEST_NAME = "test110ModifyJackDummyAccountTimeout"; + displayTestTitle(TEST_NAME); + + getDummyResource().setOperationDelayOffset(3000); + + Task task = createTask(TEST_NAME); + OperationResult result = task.getResult(); + + // WHEN + displayWhen(TEST_NAME); + + modifyUserReplace(USER_JACK_OID, UserType.F_FULL_NAME, task, result, createPolyString(USER_JACK_FULL_NAME_CAPTAIN)); + + // THEN + displayThen(TEST_NAME); + assertInProgress(result); + + assertDummyAccountByUsername(null, USER_JACK_USERNAME) + // operation timed out, data not updated + .assertFullName(USER_JACK_FULL_NAME); + } + + @Test + public void test112ModifyJackDummyAccounRetry() throws Exception { + final String TEST_NAME = "test112ModifyJackDummyAccounRetry"; + displayTestTitle(TEST_NAME); + + getDummyResource().setOperationDelayOffset(0); + clockForward("P1D"); + + Task task = createTask(TEST_NAME); + OperationResult result = task.getResult(); + + // WHEN + displayWhen(TEST_NAME); + + reconcileUser(USER_JACK_OID, task, result); + + // THEN + displayThen(TEST_NAME); + assertSuccess(result); + + assertDummyAccountByUsername(null, USER_JACK_USERNAME) + .assertFullName(USER_JACK_FULL_NAME_CAPTAIN); + } } diff --git a/testing/story/src/test/resources/logback-test.xml b/testing/story/src/test/resources/logback-test.xml index d7a372551f2..649cd0a84b1 100644 --- a/testing/story/src/test/resources/logback-test.xml +++ b/testing/story/src/test/resources/logback-test.xml @@ -82,6 +82,7 @@ + From cc622d450194c786ce7d2eb302049a508284edea Mon Sep 17 00:00:00 2001 From: Radovan Semancik Date: Thu, 7 Feb 2019 17:51:45 +0100 Subject: [PATCH 5/6] Missing test --- .../testing/story/TestLdapSyncMassive.java | 573 ++++++++++++++++++ 1 file changed, 573 insertions(+) create mode 100644 testing/story/src/test/java/com/evolveum/midpoint/testing/story/TestLdapSyncMassive.java diff --git a/testing/story/src/test/java/com/evolveum/midpoint/testing/story/TestLdapSyncMassive.java b/testing/story/src/test/java/com/evolveum/midpoint/testing/story/TestLdapSyncMassive.java new file mode 100644 index 00000000000..beed6191255 --- /dev/null +++ b/testing/story/src/test/java/com/evolveum/midpoint/testing/story/TestLdapSyncMassive.java @@ -0,0 +1,573 @@ +/* + * Copyright (c) 2016-2019 Evolveum + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.evolveum.midpoint.testing.story; + + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertNotNull; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.opends.server.types.DirectoryException; +import org.opends.server.types.Entry; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.annotation.DirtiesContext.ClassMode; +import org.springframework.test.context.ContextConfiguration; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import com.evolveum.midpoint.prism.PrismObject; +import com.evolveum.midpoint.schema.SearchResultList; +import com.evolveum.midpoint.schema.constants.MidPointConstants; +import com.evolveum.midpoint.schema.constants.SchemaConstants; +import com.evolveum.midpoint.schema.result.OperationResult; +import com.evolveum.midpoint.schema.statistics.ConnectorOperationalStatus; +import com.evolveum.midpoint.schema.util.ObjectTypeUtil; +import com.evolveum.midpoint.task.api.Task; +import com.evolveum.midpoint.test.util.MidPointTestConstants; +import com.evolveum.midpoint.test.util.ParallelTestThread; +import com.evolveum.midpoint.test.util.TestUtil; +import com.evolveum.midpoint.util.exception.CommunicationException; +import com.evolveum.midpoint.util.exception.ConfigurationException; +import com.evolveum.midpoint.util.exception.ExpressionEvaluationException; +import com.evolveum.midpoint.util.exception.ObjectAlreadyExistsException; +import com.evolveum.midpoint.util.exception.ObjectNotFoundException; +import com.evolveum.midpoint.util.exception.PolicyViolationException; +import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.util.exception.SecurityViolationException; +import com.evolveum.midpoint.xml.ns._public.common.api_types_3.ImportOptionsType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.TaskType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.UserType; + +/** + * Testing sync, with lot of sync cycles. The goal is to test thread pooling and memory + * management related to sync (e.g. MID-5099) + * + * @author Radovan Semancik + * + */ +@ContextConfiguration(locations = {"classpath:ctx-story-test-main.xml"}) +@DirtiesContext(classMode = ClassMode.AFTER_CLASS) +public class TestLdapSyncMassive extends AbstractStoryTest { + + public static final File TEST_DIR = new File(MidPointTestConstants.TEST_RESOURCES_DIR, "ldap-sync-massive"); + + private static final String RESOURCE_OPENDJ_OID = "10000000-0000-0000-0000-000000000003"; + private static final String RESOURCE_OPENDJ_NAMESPACE = MidPointConstants.NS_RI; + + private static final File RESOURCE_OPENDJ_FILE_BAD = new File(TEST_DIR, "resource-opendj-bad.xml"); + + private static final File TASK_LIVE_SYNC_FILE = new File(TEST_DIR, "task-live-sync.xml"); + private static final String TASK_LIVE_SYNC_OID = "eba4a816-2a05-11e9-9123-03a2334b9b4c"; + + private static final File ACCOUNT_WILL_LDIF_FILE = new File(TEST_DIR, "will.ldif"); + private static final String ACCOUNT_WILL_LDAP_UID = "will"; + private static final String ACCOUNT_WILL_LDAP_CN = "Will Turner"; + + private static final File ACCOUNT_KRAKEN_LDIF_FILE = new File(TEST_DIR, "kraken.ldif"); + private static final String ACCOUNT_KRAKEN_LDAP_UID = "kraken"; + private static final String ACCOUNT_KRAKEN_LDAP_CN = "Kraken Krakenoff"; + + private static final int THREAD_COUNT_TOLERANCE = 5; + private static final int THREAD_COUNT_TOLERANCE_BIG = 20; + + private static final int SYNC_ADD_ATTEMPTS = 30; + private static final int NUMBER_OF_GOBLINS = 50; + + private static final int NUMBER_OF_TEST_THREADS = 5; + private static final Integer TEST_THREADS_RANDOM_START_RANGE = 10; + private static final long PARALLEL_TEST_TIMEOUT = 60000L; + + private PrismObject resourceOpenDj; + private Integer lastSyncToken; + private int threadCountBaseline; + + private File getTestDir() { + return TEST_DIR; + } + + private File getResourceOpenDjFile() { + return new File(getTestDir(), "resource-opendj.xml"); + } + + @Override + protected void startResources() throws Exception { + openDJController.startCleanServer(); + } + + @AfterClass + public static void stopResources() throws Exception { + openDJController.stop(); + } + + @Override + public void initSystem(Task initTask, OperationResult initResult) throws Exception { + super.initSystem(initTask, initResult); + + // Resources + resourceOpenDj = importAndGetObjectFromFile(ResourceType.class, getResourceOpenDjFile(), RESOURCE_OPENDJ_OID, initTask, initResult); + openDJController.setResource(resourceOpenDj); + } + + @Test + public void test000Sanity() throws Exception { + final String TEST_NAME = "test000Sanity"; + displayTestTitle(TEST_NAME); + Task task = createTask(TEST_NAME); + + OperationResult testResultOpenDj = modelService.testResource(RESOURCE_OPENDJ_OID, task); + TestUtil.assertSuccess(testResultOpenDj); + + assertLdapConnectorInstances(1); + + dumpLdap(); + } + + @Test + public void test080ImportSyncTask() throws Exception { + final String TEST_NAME = "test080ImportSyncTask"; + displayTestTitle(TEST_NAME); + + // WHEN + displayWhen(TEST_NAME); + + importObjectFromFile(TASK_LIVE_SYNC_FILE); + + // THEN + displayThen(TEST_NAME); + + waitForTaskNextRunAssertSuccess(TASK_LIVE_SYNC_OID, true); + + PrismObject syncTask = getTask(TASK_LIVE_SYNC_OID); + lastSyncToken = ObjectTypeUtil.getExtensionItemRealValue(syncTask, SchemaConstants.SYNC_TOKEN); + display("Initial sync token", lastSyncToken); + assertNotNull("Null sync token", lastSyncToken); + + assertLdapConnectorInstances(1); + + threadCountBaseline = Thread.activeCount(); + display("Thread count baseline", threadCountBaseline); + + dumpLdap(); + } + + /** + * Add a single LDAP account. This goal is to test whether we have good configuration. + */ + @Test + public void test110SyncAddWill() throws Exception { + final String TEST_NAME = "test110SyncAddWill"; + displayTestTitle(TEST_NAME); + + Entry entry = openDJController.addEntryFromLdifFile(ACCOUNT_WILL_LDIF_FILE); + display("Entry from LDIF", entry); + + // WHEN + displayWhen(TEST_NAME); + + waitForTaskNextRunAssertSuccess(TASK_LIVE_SYNC_OID, true); + + // THEN + displayThen(TEST_NAME); + + assertSyncTokenIncrement(1); + + assertLdapConnectorInstances(1); + + assertUserAfterByUsername(ACCOUNT_WILL_LDAP_UID) + .assertFullName(ACCOUNT_WILL_LDAP_CN); + + assertThreadCount(); + + // just to make sure we are stable + + waitForTaskNextRunAssertSuccess(TASK_LIVE_SYNC_OID, true); + + assertSyncTokenIncrement(0); + assertLdapConnectorInstances(1); + assertThreadCount(); + + dumpLdap(); + + } + + /** + * "Good run". This is a run with more sync cycles, but without + * any effort to trigger problems. This is here to make sure we + * have the right "baseline", e.g. thread count tolerance. + */ + @Test + public void test112SyncAddGoods() throws Exception { + final String TEST_NAME = "test112SyncAddGoods"; + displayTestTitle(TEST_NAME); + + // WHEN + displayWhen(TEST_NAME); + + for (int i = 0; i < SYNC_ADD_ATTEMPTS; i++) { + syncAddAttemptGood("good", i); + } + + // THEN + displayThen(TEST_NAME); + + dumpLdap(); + + } + + + /** + * Add "goblin" users, each with an LDAP account. + * We do not really needs them now. But these will make + * subsequent tests more massive. + * Adding them in this way is much faster then adding + * them in sync one by one. + * And we need to add them while the resource still + * works OK. + */ + @Test + public void test150AddGoblins() throws Exception { + final String TEST_NAME = "test150AddGoblins"; + displayTestTitle(TEST_NAME); + + // WHEN + displayWhen(TEST_NAME); + + for (int i = 0; i < NUMBER_OF_GOBLINS; i++) { + String username = goblinUsername(i); + PrismObject goblin = createUser(username, "Goblin", Integer.toString(i), true); + goblin.asObjectable(). + beginAssignment() + .beginConstruction() + .resourceRef(RESOURCE_OPENDJ_OID, ResourceType.COMPLEX_TYPE); + addObject(goblin); + } + + // THEN + displayThen(TEST_NAME); + + dumpLdap(); + assertLdapConnectorInstances(1,2); + + waitForTaskNextRunAssertSuccess(TASK_LIVE_SYNC_OID, true); + + assertLdapConnectorInstances(1,2); + assertSyncTokenIncrement(NUMBER_OF_GOBLINS); + assertThreadCount(); + + waitForTaskNextRunAssertSuccess(TASK_LIVE_SYNC_OID, true); + + assertLdapConnectorInstances(1,2); + assertSyncTokenIncrement(0); + assertThreadCount(); + + } + + + + private String goblinUsername(int i) { + return String.format("goblin%05d", i); + } + + /** + * Overwrite the resource with a bad configuration. + * Now we are going to make some trouble. + */ + @Test + public void test200SyncAddKraken() throws Exception { + final String TEST_NAME = "test200SyncAddKraken"; + displayTestTitle(TEST_NAME); + + Task task = createTask(TEST_NAME); + OperationResult result = task.getResult(); + + ImportOptionsType options = new ImportOptionsType() + .overwrite(true); + importObjectFromFile(RESOURCE_OPENDJ_FILE_BAD, options, task, result); + + OperationResult testResultOpenDj = modelService.testResource(RESOURCE_OPENDJ_OID, task); + display("Test resource result", testResultOpenDj); + TestUtil.assertSuccess(testResultOpenDj); + + PrismObject resourceAfter = modelService.getObject(ResourceType.class, RESOURCE_OPENDJ_OID, null, task, result); + assertResource(resourceAfter, "after") + .assertHasSchema(); + + assertLdapConnectorInstances(1,2); + } + + /** + * Just make first attempt with bad configuration. + * This is here mostly to make sure we really have a bad configuration. + */ + @Test + public void test210SyncAddKraken() throws Exception { + final String TEST_NAME = "test210SyncAddKraken"; + displayTestTitle(TEST_NAME); + + Entry entry = openDJController.addEntryFromLdifFile(ACCOUNT_KRAKEN_LDIF_FILE); + display("Entry from LDIF", entry); + + // WHEN + displayWhen(TEST_NAME); + + OperationResult taskResult = waitForTaskNextRun(TASK_LIVE_SYNC_OID); + + // THEN + displayThen(TEST_NAME); + assertPartialError(taskResult); + + assertSyncTokenIncrement(0); + assertLdapConnectorInstances(1,2); + assertThreadCount(); + + // just to make sure we are stable + // in fact, it is "FUBAR, but stable" + + taskResult = waitForTaskNextRun(TASK_LIVE_SYNC_OID); + assertPartialError(taskResult); + + assertSyncTokenIncrement(0); + assertLdapConnectorInstances(1,2); + assertThreadCount(); + + dumpLdap(); + + } + + /** + * "Bad run". + * MID-5099: cannot reproduce + */ + @Test + public void test212SyncAddBads() throws Exception { + final String TEST_NAME = "test212SyncAddBads"; + displayTestTitle(TEST_NAME); + + // WHEN + displayWhen(TEST_NAME); + + for (int i = 0; i < SYNC_ADD_ATTEMPTS; i++) { + syncAddAttemptBad("bad", i); + } + + // THEN + displayThen(TEST_NAME); + + dumpLdap(); + + } + + /** + * Suspend sync task. We do not want that to mess the results of subsequent + * tests (e.g. mess the number of connector instances). + */ + @Test + public void test219StopSyncTask() throws Exception { + final String TEST_NAME = "test219StopSyncTask"; + displayTestTitle(TEST_NAME); + + // WHEN + displayWhen(TEST_NAME); + + suspendTask(TASK_LIVE_SYNC_OID); + + // THEN + displayThen(TEST_NAME); + + assertSyncTokenIncrement(0); + assertLdapConnectorInstances(1,2); + assertThreadCount(); + + } + + @Test + public void test230UserRecomputeSequential() throws Exception { + final String TEST_NAME = "test230UserRecomputeSequential"; + displayTestTitle(TEST_NAME); + + Task task = createTask(TEST_NAME); + OperationResult result = task.getResult(); + + SearchResultList> users = modelService.searchObjects(UserType.class, null, null, task, result); + + // WHEN + displayWhen(TEST_NAME); + + for (PrismObject user : users) { + reconcile(TEST_NAME, user); + } + + // THEN + displayThen(TEST_NAME); + + assertLdapConnectorInstances(1,2); + assertThreadCount(); + } + + @Test + public void test232UserRecomputeParallel() throws Exception { + final String TEST_NAME = "test232UserRecomputeParallel"; + displayTestTitle(TEST_NAME); + + Task task = createTask(TEST_NAME); + OperationResult result = task.getResult(); + + SearchResultList> users = modelService.searchObjects(UserType.class, null, null, task, result); + + // WHEN + displayWhen(TEST_NAME); + + int segmentSize = users.size() / NUMBER_OF_TEST_THREADS; + ParallelTestThread[] threads = multithread(TEST_NAME, + (threadIndex) -> { + for (int i = segmentSize * threadIndex; i < segmentSize * threadIndex + segmentSize; i++) { + PrismObject user = users.get(i); + reconcile(TEST_NAME, user); + } + + }, NUMBER_OF_TEST_THREADS, TEST_THREADS_RANDOM_START_RANGE); + + // THEN + displayThen(TEST_NAME); + waitForThreads(threads, PARALLEL_TEST_TIMEOUT); + + // When system is put under load, this means more threads. But not huge number of threads. + assertThreadCount(THREAD_COUNT_TOLERANCE_BIG); + assertLdapConnectorInstances(1,NUMBER_OF_TEST_THREADS); + } + + private void reconcile(final String TEST_NAME, PrismObject user) throws CommunicationException, ObjectAlreadyExistsException, ExpressionEvaluationException, PolicyViolationException, SchemaException, SecurityViolationException, ConfigurationException, ObjectNotFoundException { + Task task = createTask(TEST_NAME+".user."+user.getName()); + OperationResult result = task.getResult(); + + reconcileUser(user.getOid(), task, result); + + // We do not bother to check result. Even though the + // timeout is small, the operation may succeed occasionally. + // This annoying success cout cause the tests to fail. + } + + private void syncAddAttemptGood(String prefix, int index) throws Exception { + + String uid = String.format("%s%05d", prefix, index); + String cn = prefix+" "+index; + addAttemptEntry(uid, cn, Integer.toString(index)); + + waitForTaskNextRunAssertSuccess(TASK_LIVE_SYNC_OID, true); + + assertSyncTokenIncrement(1); + + assertUserAfterByUsername(uid) + .assertFullName(cn); + + assertThreadCount(); + } + + private void syncAddAttemptBad(String prefix, int index) throws Exception { + + String uid = String.format("%s%05d", prefix, index); + String cn = prefix+" "+index; + addAttemptEntry(uid, cn, Integer.toString(index)); + + OperationResult taskResult = waitForTaskNextRun(TASK_LIVE_SYNC_OID); + + assertPartialError(taskResult); + assertSyncTokenIncrement(0); + assertLdapConnectorInstances(1); + assertThreadCount(); + } + + private void addAttemptEntry(String uid, String cn, String sn) throws Exception { + Entry entry = openDJController.addEntry( + "dn: uid="+uid+",ou=People,dc=example,dc=com\n" + + "uid: "+uid+"\n" + + "cn: "+cn+"\n" + + "sn: "+sn+"\n" + + "givenname: "+uid+"\n" + + "objectclass: top\n" + + "objectclass: person\n" + + "objectclass: organizationalPerson\n" + + "objectclass: inetOrgPerson" + ); + display("Added generated entry", entry); + } + + private void assertThreadCount() { + assertThreadCount(THREAD_COUNT_TOLERANCE); + } + + private void assertThreadCount(int tolerance) { + int currentThreadCount = Thread.activeCount(); + if (!isWithinTolerance(threadCountBaseline, currentThreadCount, tolerance)) { + fail("Thread count out of tolerance: "+currentThreadCount+" ("+(currentThreadCount-threadCountBaseline)+")"); + } + } + + private boolean isWithinTolerance(int baseline, int currentCount, int tolerance) { + if (currentCount > baseline + tolerance) { + return false; + } + if (currentCount < baseline - tolerance) { + return false; + } + return true; + } + + private void assertSyncTokenIncrement(int expectedIncrement) throws ObjectNotFoundException, SchemaException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException { + PrismObject syncTask = getTask(TASK_LIVE_SYNC_OID); + Integer currentSyncToken = ObjectTypeUtil.getExtensionItemRealValue(syncTask, SchemaConstants.SYNC_TOKEN); + display("Sync token, last="+lastSyncToken+", current="+currentSyncToken+", expectedIncrement="+expectedIncrement); + if (currentSyncToken != lastSyncToken + expectedIncrement) { + fail("Expected sync token increment "+expectedIncrement+", but it was "+(currentSyncToken-lastSyncToken)); + } + lastSyncToken = currentSyncToken; + } + + protected void assertLdapConnectorInstances(int expectedConnectorInstances) throws NumberFormatException, IOException, InterruptedException, SchemaException, ObjectNotFoundException, CommunicationException, ConfigurationException, ExpressionEvaluationException { + assertLdapConnectorInstances(expectedConnectorInstances, expectedConnectorInstances); + } + + protected void assertLdapConnectorInstances(int expectedConnectorInstancesMin, int expectedConnectorInstancesMax) throws NumberFormatException, IOException, InterruptedException, SchemaException, ObjectNotFoundException, CommunicationException, ConfigurationException, ExpressionEvaluationException { + Task task = createTask(TestLdapSyncMassive.class.getName() + ".assertLdapConnectorInstances"); + OperationResult result = task.getResult(); + List stats = provisioningService.getConnectorOperationalStatus(RESOURCE_OPENDJ_OID, task, result); + display("Resource connector stats", stats); + assertSuccess(result); + + assertEquals("unexpected number of stats", 1, stats.size()); + ConnectorOperationalStatus stat = stats.get(0); + + int actualConnectorInstances = stat.getPoolStatusNumIdle() + stat.getPoolStatusNumActive(); + + if (actualConnectorInstances < expectedConnectorInstancesMin) { + fail("Number of LDAP connector instances too low: "+actualConnectorInstances+", expected at least "+expectedConnectorInstancesMin); + } + if (actualConnectorInstances > expectedConnectorInstancesMax) { + fail("Number of LDAP connector instances too high: "+actualConnectorInstances+", expected at most "+expectedConnectorInstancesMax); + } + } + + private void dumpLdap() throws DirectoryException { + display("LDAP server tree", openDJController.dumpTree()); + } + + +} \ No newline at end of file From df6b34aedd77341c4f59f2f0947f530fa466c4eb Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Thu, 7 Feb 2019 18:39:38 +0100 Subject: [PATCH 6/6] Make DeleteTaskHandler stoppable (MID-5134) Originally this task handler did not watch for task.canRun() as it should. --- .../model/impl/util/DeleteTaskHandler.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/util/DeleteTaskHandler.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/util/DeleteTaskHandler.java index 7a32ea12e69..74dad692d1e 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/util/DeleteTaskHandler.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/util/DeleteTaskHandler.java @@ -189,7 +189,7 @@ private TaskRunResult runInternal(Task task) { } SearchResultList> objects; - while (true) { + while (task.canRun()) { objects = modelService.searchObjects(objectType, query, searchOptions, task, opResult); if (objects.isEmpty()) { break; @@ -197,6 +197,9 @@ private TaskRunResult runInternal(Task task) { int skipped = 0; for (PrismObject object: objects) { + if (!task.canRun()) { + break; + } if (!optionRaw && ShadowType.class.isAssignableFrom(objectType) && isTrue(((ShadowType)(object.asObjectable())).isProtectedObject())) { LOGGER.debug("Skipping delete of protected object {}", object); @@ -204,8 +207,7 @@ && isTrue(((ShadowType)(object.asObjectable())).isProtectedObject())) { continue; } - ObjectDelta delta = prismContext.deltaFactory().object().createDeleteDelta(objectType, object.getOid() - ); + ObjectDelta delta = prismContext.deltaFactory().object().createDeleteDelta(objectType, object.getOid()); String objectName = PolyString.getOrig(object.getName()); String objectDisplayName = StatisticsUtil.getDisplayName(object); @@ -225,14 +227,13 @@ && isTrue(((ShadowType)(object.asObjectable())).isProtectedObject())) { opResult.summarize(); if (LOGGER.isTraceEnabled()) { - LOGGER.trace("Search returned {} objects, {} skipped, progress: {}, result:\n{}", - objects.size(), skipped, task.getProgress(), opResult.debugDump()); + LOGGER.trace("Search returned {} objects, {} skipped, progress: {} (interrupted: {}), result:\n{}", + objects.size(), skipped, task.getProgress(), !task.canRun(), opResult.debugDump()); } if (objects.size() == skipped) { break; } - } } catch (ObjectAlreadyExistsException | ObjectNotFoundException | SchemaException @@ -260,11 +261,14 @@ && isTrue(((ShadowType)(object.asObjectable())).isProtectedObject())) { if (task.getProgress() > 0) { statistics += " Wall clock time average: " + ((float) wallTime / (float) task.getProgress()) + " milliseconds"; } + if (!task.canRun()) { + statistics += " (task run was interrupted)"; + } opResult.createSubresult(DeleteTaskHandler.class.getName() + ".statistics").recordStatus(OperationResultStatus.SUCCESS, statistics); LOGGER.info(finishMessage + statistics); - LOGGER.trace("Run finished (task {}, run result {})", task, runResult); + LOGGER.trace("Run finished (task {}, run result {}); interrupted = {}", task, runResult, !task.canRun()); return runResult; }