Add bulk import performance test (MID-5368)

model/model-impl/src/test/java/com/evolveum/midpoint/model/impl/AbstractInternalModelIntegrationTest.java

@@ -134,8 +134,8 @@ public void initSystem(Task initTask, OperationResult initResult) throws Excepti
 		LOGGER.trace("initSystem");
 		super.initSystem(initTask, initResult);
 
-		// We want logging config from logback-test.xml and not from system config object
-		InternalsConfig.setAvoidLoggingChange(true);
+		// We want logging config from logback-test.xml and not from system config object (unless suppressed)
+		InternalsConfig.setAvoidLoggingChange(isAvoidLoggingChange());
 
 		mockClockworkHook = new MockClockworkHook();
 		hookRegistry.registerChangeHook(MOCK_CLOCKWORK_HOOK_URL, mockClockworkHook);

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/AbstractConfiguredModelIntegrationTest.java

@@ -551,8 +551,9 @@ public AbstractConfiguredModelIntegrationTest() {
 	@Override
 	public void initSystem(Task initTask,  OperationResult initResult) throws Exception {
 		LOGGER.trace("initSystem");
-		// We want logging config from logback-test.xml and not from system config object
-		InternalsConfig.setAvoidLoggingChange(true);
+
+		// We want logging config from logback-test.xml and not from system config object (unless suppressed)
+		InternalsConfig.setAvoidLoggingChange(isAvoidLoggingChange());
 		super.initSystem(initTask, initResult);
 
 		modelService.postInit(initResult);

model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/password/AbstractPasswordTest.java

@@ -143,7 +143,7 @@ public abstract class AbstractPasswordTest extends AbstractInitializedModelInteg
 
 	@Override
 	public void initSystem(Task initTask, OperationResult initResult) throws Exception {
-		InternalsConfig.setAvoidLoggingChange(true);
+		InternalsConfig.setAvoidLoggingChange(isAvoidLoggingChange());
 		super.initSystem(initTask, initResult);
 
 		importObjectFromFile(PASSWORD_POLICY_UGLY_FILE);

model/model-test/src/main/java/com/evolveum/midpoint/model/test/AbstractModelIntegrationTest.java

@@ -321,7 +321,7 @@ public void initSystem(Task initTask, OperationResult initResult) throws Excepti
 		startResources();
 		dummyAuditService = DummyAuditService.getInstance();
 		InternalsConfig.reset();
-		InternalsConfig.setAvoidLoggingChange(true);
+		InternalsConfig.setAvoidLoggingChange(isAvoidLoggingChange());
 		// Make sure the checks are turned on
 		InternalsConfig.turnOnAllChecks();
         // By default, notifications are turned off because of performance implications. Individual tests turn them on for themselves.
@@ -330,6 +330,10 @@ public void initSystem(Task initTask, OperationResult initResult) throws Excepti
         }
 	}
 
+	protected boolean isAvoidLoggingChange() {
+		return true;
+	}
+
 	@Override
 	public void postInitSystem(Task initTask, OperationResult initResult) throws Exception {
 		super.postInitSystem(initTask, initResult);

testing/story/pom.xml

@@ -303,6 +303,12 @@
             <artifactId>api-all</artifactId>
             <scope>test</scope>
         </dependency>
+		<dependency>
+			<groupId>mysql</groupId>
+			<artifactId>mysql-connector-java</artifactId>
+			<version>${jdbc.mysql}</version>
+			<scope>test</scope>
+		</dependency>
 	</dependencies>
 	<profiles>
 		<profile>

testing/story/src/test/java/com/evolveum/midpoint/testing/story/perf/Generate.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2010-2019 Evolveum
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.evolveum.midpoint.testing.story.perf;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ * Generates data for measuring DB-to-midPoint import performance. See MID-5368.
+ *
+ * Format is like this:
+ *
+ * INSERT INTO `people` VALUES('fzum_00000000', 'Gfzum_00000000', 'Sfzum_00000000', 'fzum_00000000@test.edu', '9981283310',
+ *      '2019-05-27 14:40:29', NULL, 'CIOXP', 0, 'DXMQW', NULL, 'ShjXu@test.edu',
+ *      'DXMQW|VCUKx|EeNkQ|hdRPK|hQcSa|hGDBQ|FdmHk|mYpIt|fdiiT', 'ITPfv', 'f_ReAUQfIKHI', 0, NULL, '2019-05-27');
+ *
+ */
+public class Generate {
+
+	private static final int USERS = 50000;
+
+	public static void main(String[] args) throws IOException {
+		SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US);
+		SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US);
+
+		PrintWriter pw = new PrintWriter(new FileWriter("target/source-data.sql"));
+
+		for (int i = 0; i < USERS; i++) {
+			String login = String.format("%s_%08d", RandomStringUtils.randomAlphabetic(4), i);
+			String employeeNumber = RandomStringUtils.randomNumeric(10);
+			String activation = dateTimeFormat.format(new Date());
+			String department = RandomStringUtils.randomAlphabetic(5);
+			List<String> roles = generateValues(10);
+			List<String> services = generateValues(3);
+			List<String> aliases = generateValues(3);
+			String facultyCode = "f_" + RandomStringUtils.randomAlphabetic(10);
+
+			pw.print(String.format("INSERT INTO `people` VALUES('%s', '%s', '%s', '%s@test.edu', '%s', ",
+					login, "G"+login, "S"+login, login, employeeNumber));
+			pw.print(String.format("'%s', NULL, '%s', %d, '%s', NULL, '%s', '%s', '%s', '%s', ", activation, department, i,
+					roles.get(0), concat(aliases, "@test.edu"), concat(roles, ""), concat(services, ""), facultyCode));
+			pw.println(String.format("%d, NULL, '%s');", i, dateFormat.format(new Date())));
+		}
+
+		pw.close();
+	}
+
+	private static String concat(List<String> strings, String suffix) {
+		StringBuilder sb = new StringBuilder();
+		boolean first = true;
+		for (String string : strings) {
+			if (first) {
+				first = false;
+			} else {
+				sb.append("|");
+			}
+			sb.append(string).append(suffix);
+		}
+		return sb.toString();
+	}
+
+	@NotNull
+	private static List<String> generateValues(int limit) {
+		List<String> rv = new ArrayList<>();
+		int r = (int) (Math.random() * limit) + 1;
+		while (r-- > 0) {
+			rv.add(RandomStringUtils.randomAlphabetic(5));
+		}
+		return rv;
+	}
+}

testing/story/src/test/java/com/evolveum/midpoint/testing/story/perf/TestImport.java

@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2010-2019 Evolveum
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.evolveum.midpoint.testing.story.perf;
+
+import com.evolveum.midpoint.prism.PrismObject;
+import com.evolveum.midpoint.schema.internals.InternalsConfig;
+import com.evolveum.midpoint.schema.result.OperationResult;
+import com.evolveum.midpoint.task.api.Task;
+import com.evolveum.midpoint.test.util.MidPointTestConstants;
+import com.evolveum.midpoint.test.util.TestUtil;
+import com.evolveum.midpoint.testing.story.AbstractStoryTest;
+import com.evolveum.midpoint.testing.story.TestTrafo;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
+import com.mysql.cj.jdbc.Driver;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.annotation.DirtiesContext.ClassMode;
+import org.springframework.test.context.ContextConfiguration;
+import org.testng.annotations.Test;
+
+import java.io.File;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertNotNull;
+
+/**
+ * Tests the performance of bulk import of data. See MID-5368.
+ *
+ * This test is not meant to be run automatically.
+ * It requires externally-configured MySQL database with the data to be imported.
+ */
+@ContextConfiguration(locations = {"classpath:ctx-story-test-main.xml"})
+@DirtiesContext(classMode = ClassMode.AFTER_CLASS)
+public class TestImport extends AbstractStoryTest {
+
+	public static final File TEST_DIR = new File(MidPointTestConstants.TEST_RESOURCES_DIR, "perf/import");
+
+	public static final File SYSTEM_CONFIGURATION_FILE = new File(TEST_DIR, "system-configuration.xml");
+
+	public static final File OBJECT_TEMPLATE_FILE = new File(TEST_DIR, "template-import.xml");
+	public static final String OBJECT_TEMPLATE_OID = "e84d7b5a-4634-4b75-a17c-df0b8b49b593";
+
+	protected static final File RESOURCE_SOURCE_FILE = new File(TEST_DIR, "resource-source.xml");
+	protected static final String RESOURCE_SOURCE_OID = "f2dd9222-6aff-4099-b5a2-04ae6b3a00b7";
+
+	protected static final File ORG_BASIC_FILE = new File(TEST_DIR, "org-basic.xml");
+
+	protected static final File TASK_IMPORT_FILE = new File(TEST_DIR, "task-import.xml");
+	protected static final String TASK_IMPORT_OID = "50142510-8003-4a47-993a-2434119f5028";
+
+	private static final int IMPORT_TIMEOUT = 3600_000 * 3;     // 3 hours
+
+	private static final int USERS = 50000;
+
+	private int usersBefore;
+
+	private PrismObject<ResourceType> sourceResource;
+
+	@Override
+	public void initSystem(Task initTask, OperationResult initResult) throws Exception {
+		super.initSystem(initTask, initResult);
+
+		InternalsConfig.turnOffAllChecks();
+
+		Class.forName(Driver.class.getName());
+
+		// Resources
+		sourceResource = importAndGetObjectFromFile(ResourceType.class, RESOURCE_SOURCE_FILE, RESOURCE_SOURCE_OID, initTask, initResult);
+
+		// Object Templates
+		importObjectFromFile(OBJECT_TEMPLATE_FILE, initResult);
+		setDefaultUserTemplate(OBJECT_TEMPLATE_OID);
+
+		// Org
+		repoAddObjectFromFile(ORG_BASIC_FILE, OrgType.class, initResult);
+
+		usersBefore = repositoryService.countObjects(UserType.class, null, null, initResult);
+		display("users before", usersBefore);
+
+		//InternalMonitor.setTrace(InternalOperationClasses.PRISM_OBJECT_CLONES, true);
+	}
+
+	@Override
+	protected boolean isAvoidLoggingChange() {
+		return false;           // we want logging from our system config
+	}
+
+	@Override
+	protected File getSystemConfigurationFile() {
+		return SYSTEM_CONFIGURATION_FILE;
+	}
+
+	@Override
+	protected void importSystemTasks(OperationResult initResult) {
+		// nothing here
+	}
+
+	@Test
+    public void test000Sanity() throws Exception {
+		final String TEST_NAME = "test000Sanity";
+        displayTestTitle(TEST_NAME);
+        Task task = taskManager.createTaskInstance(TestTrafo.class.getName() + "." + TEST_NAME);
+
+        OperationResult testResultHr = modelService.testResource(RESOURCE_SOURCE_OID, task);
+        TestUtil.assertSuccess(testResultHr);
+
+        SystemConfigurationType systemConfiguration = getSystemConfiguration();
+        assertNotNull("No system configuration", systemConfiguration);
+        display("System config", systemConfiguration);
+	}
+
+	@Test
+    public void test100RunImport() throws Exception {
+		final String TEST_NAME = "test100RunImport";
+        displayTestTitle(TEST_NAME);
+        Task task = taskManager.createTaskInstance(TestTrafo.class.getName() + "." + TEST_NAME);
+        OperationResult result = task.getResult();
+
+        // WHEN
+        TestUtil.displayWhen(TEST_NAME);
+		importObjectFromFile(TASK_IMPORT_FILE, result);
+
+        // THEN
+        TestUtil.displayThen(TEST_NAME);
+        waitForTaskFinish(TASK_IMPORT_OID, true, IMPORT_TIMEOUT);
+
+        result.computeStatus();
+        TestUtil.assertSuccess(result);
+
+		PrismObject<TaskType> taskAfter = repositoryService.getObject(TaskType.class, TASK_IMPORT_OID, null, result);
+		String taskXml = prismContext.xmlSerializer().serialize(taskAfter);
+		display("Task after", taskXml);
+
+		int usersAfter = repositoryService.countObjects(UserType.class, null, null, result);
+		display("users after", usersAfter);
+		assertEquals("Wrong # of users", usersBefore + USERS, usersAfter);
+
+	}
+
+}

testing/story/src/test/resources/perf/import/org-basic.xml

@@ -0,0 +1,20 @@
+<!--
+  ~ Copyright (c) 2010-2019 Evolveum
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<org xmlns="http://midpoint.evolveum.com/xml/ns/public/common/common-3"
+	 oid="2c8fdf1e-929e-45bf-a750-7822f5cc6de4">
+	<name>basic</name>
+</org>