report-impl: overhaul of TestCsvReportAllAssignments, much faster report

- report-indirect-assignments.xml is now run with new task that uses custom caching profile (task-export-role-caching.xml) - number of getObject called on the physical repo is drastically reduced - more work done in the "data" subreport, column scripts are simpler - also, report resolves objects on repo directly, because model is currently very slow with SchemaTransformer.applySchemaAndSecurity()
Evolveum · Feb 17, 2023 · b7f9477 · b7f9477
1 parent f7925fc
commit b7f9477
Show file tree

Hide file tree

Showing 7 changed files with 171 additions and 69 deletions.
diff --git a/model/report-impl/src/test/java/com/evolveum/midpoint/report/EmptyReportIntegrationTest.java b/model/report-impl/src/test/java/com/evolveum/midpoint/report/EmptyReportIntegrationTest.java
@@ -1,34 +1,33 @@
 /*
- * Copyright (C) 2010-2022 Evolveum and contributors
+ * Copyright (C) 2010-2023 Evolveum and contributors
  *
  * This work is dual-licensed under the Apache License 2.0
  * and European Union Public License. See LICENSE file for details.
  */
 package com.evolveum.midpoint.report;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.testng.AssertJUnit.assertNotNull;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
 
-import com.evolveum.midpoint.prism.*;
-
-import com.evolveum.midpoint.test.TestObject;
-
 import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.test.context.ContextConfiguration;
 
 import com.evolveum.midpoint.model.test.AbstractModelIntegrationTest;
 import com.evolveum.midpoint.notifications.api.transports.Message;
+import com.evolveum.midpoint.prism.PrismObject;
 import com.evolveum.midpoint.prism.crypto.EncryptionException;
 import com.evolveum.midpoint.prism.path.ItemName;
 import com.evolveum.midpoint.prism.path.ItemPath;
 import com.evolveum.midpoint.repo.api.RepoAddOptions;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.task.api.Task;
-import com.evolveum.midpoint.util.exception.*;
+import com.evolveum.midpoint.test.TestObject;
+import com.evolveum.midpoint.test.TestTask;
+import com.evolveum.midpoint.util.exception.CommonException;
+import com.evolveum.midpoint.util.exception.ObjectAlreadyExistsException;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
 
 /**
@@ -40,7 +39,8 @@ public abstract class EmptyReportIntegrationTest extends AbstractModelIntegratio
 
     static final int DONT_COUNT_ROWS = -1;
 
-    static final File TEST_DIR_REPORTS = new File("src/test/resources/reports");
+    static final String DIR_REPORTS = "reports";
+    static final File TEST_DIR_REPORTS = new File("src/test/resources/" + DIR_REPORTS);
     static final File TEST_DIR_COMMON = new File("src/test/resources/common");
     private static final File EXPORT_DIR = new File("target/midpoint-home/export");
 
@@ -139,7 +139,7 @@ public abstract class EmptyReportIntegrationTest extends AbstractModelIntegratio
     static final TestObject<ObjectCollectionType> USER_JACK = TestObject.file(TEST_DIR_COMMON,
             "user-jack.xml", "c0c010c0-d34d-b33f-f00d-111111111111");
 
-    static final TestObject<TaskType> TASK_EXPORT_CLASSIC = TestObject.file(TEST_DIR_REPORTS,
+    static final TestTask TASK_EXPORT_CLASSIC = TestTask.file(TEST_DIR_REPORTS,
             "task-export.xml", "d3a13f2e-a8c0-4f8c-bbf9-e8996848bddf");
 
     private static final TestObject<ArchetypeType> ARCHETYPE_TASK_REPORT_EXPORT_CLASSIC = TestObject.file(TEST_DIR_COMMON,
@@ -212,23 +212,21 @@ void runExportTaskClassic(TestObject<ReportType> reportResource, OperationResult
         runExportTask(TASK_EXPORT_CLASSIC, reportResource, result);
     }
 
-    void runExportTask(TestObject<TaskType> taskResource, TestObject<ReportType> reportResource, OperationResult result)
+    void runExportTask(TestObject<TaskType> testTask, TestObject<ReportType> testResource, OperationResult result)
             throws CommonException {
         modifyObjectReplaceContainer(ReportType.class,
-                reportResource.oid,
+                testResource.oid,
                 ReportType.F_FILE_FORMAT,
                 getTestTask(),
                 result,
-                getFileFormatConfiguration()
-        );
-        changeTaskReport(reportResource,
+                getFileFormatConfiguration());
+        changeTaskReport(testResource,
                 ItemPath.create(TaskType.F_ACTIVITY,
                         ActivityDefinitionType.F_WORK,
                         getWorkDefinitionType(),
-                        ClassicReportImportWorkDefinitionType.F_REPORT_REF
-                ),
-                taskResource);
-        rerunTask(taskResource.oid, result);
+                        ClassicReportImportWorkDefinitionType.F_REPORT_REF),
+                testTask);
+        rerunTask(testTask.oid, result);
     }
 
     protected ItemName getWorkDefinitionType() {
@@ -241,9 +239,9 @@ void assertNotificationMessage(ReportType report, String expectedContentType) {
         displayDumpable("dummy transport", dummyTransport);
 
         String reportName = report.getName().getOrig();
-        assertSingleDummyTransportMessageContaining("reports", "Report: " + reportName);
+        assertSingleDummyTransportMessageContaining(DIR_REPORTS, "Report: " + reportName);
 
-        Message message = dummyTransport.getMessages("dummy:reports").get(0);
+        Message message = dummyTransport.getMessages("dummy:" + DIR_REPORTS).get(0);
         List<NotificationMessageAttachmentType> attachments = message.getAttachments();
         assertThat(attachments).as("notification message attachments").hasSize(1);
         NotificationMessageAttachmentType attachment = attachments.get(0);

diff --git a/model/report-impl/src/test/java/com/evolveum/midpoint/report/TestCsvReport.java b/model/report-impl/src/test/java/com/evolveum/midpoint/report/TestCsvReport.java
@@ -18,6 +18,7 @@
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.task.api.Task;
 import com.evolveum.midpoint.test.TestObject;
+import com.evolveum.midpoint.test.TestTask;
 import com.evolveum.midpoint.util.exception.ObjectNotFoundException;
 import com.evolveum.midpoint.util.exception.SchemaException;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
@@ -81,16 +82,28 @@ void assertNotificationMessage(TestObject<ReportType> reportTestResource) {
         assertNotificationMessage(reportTestResource.getObjectable(), "text/csv");
     }
 
-    void testClassicExport(
-            TestObject<ReportType> reportResource, int expectedRows, int expectedColumns,
+    protected void testClassicExport(TestObject<ReportType> testReport,
+            int expectedRows, int expectedColumns, String lastLine)
+            throws Exception {
+        testClassicExport(testReport, expectedRows, expectedColumns, lastLine, null);
+    }
+
+    protected void testClassicExport(
+            TestObject<ReportType> testReport, int expectedRows, int expectedColumns,
             String lastLine, ReportParameterType parameters) throws Exception {
+        testExport(TASK_EXPORT_CLASSIC, testReport, expectedRows, expectedColumns, lastLine, parameters);
+    }
+
+    protected void testExport(
+            TestTask testReportTask, TestObject<ReportType> testReport, int expectedRows,
+            int expectedColumns, String lastLine, ReportParameterType parameters) throws Exception {
         given();
         Task task = getTestTask();
         OperationResult result = task.getResult();
 
         if (parameters != null) {
             modifyObjectReplaceContainer(TaskType.class,
-                    TASK_EXPORT_CLASSIC.oid,
+                    testReportTask.oid,
                     ItemPath.create(TaskType.F_ACTIVITY,
                             ActivityDefinitionType.F_WORK,
                             WorkDefinitionsType.F_REPORT_EXPORT,
@@ -103,24 +116,18 @@ void testClassicExport(
         dummyTransport.clearMessages();
 
         when();
-        runExportTaskClassic(reportResource, result);
-        waitForTaskCloseOrSuspend(TASK_EXPORT_CLASSIC.oid);
+        runExportTask(testReportTask, testReport, result);
+        waitForTaskCloseOrSuspend(testReportTask.oid);
 
         then();
-        assertTask(TASK_EXPORT_CLASSIC.oid, "after")
+        assertTask(testReportTask.oid, "after")
                 .assertSuccess()
                 .display()
                 .assertHasArchetype(SystemObjectsType.ARCHETYPE_REPORT_EXPORT_CLASSIC_TASK.value());
 
-        PrismObject<TaskType> reportTask = getObject(TaskType.class, TASK_EXPORT_CLASSIC.oid);
+        PrismObject<TaskType> reportTask = getObject(TaskType.class, testReportTask.oid);
         basicCheckOutputFile(reportTask, expectedRows, expectedColumns, lastLine);
 
-        assertNotificationMessage(reportResource);
-    }
-
-    void testClassicExport(TestObject<ReportType> reportResource,
-            int expectedRows, int expectedColumns, String lastLine)
-            throws Exception {
-        testClassicExport(reportResource, expectedRows, expectedColumns, lastLine, null);
+        assertNotificationMessage(testReport);
     }
 }
diff --git a/...l/report-impl/src/test/java/com/evolveum/midpoint/report/TestCsvReportAllAssignments.java b/...l/report-impl/src/test/java/com/evolveum/midpoint/report/TestCsvReportAllAssignments.java
@@ -13,12 +13,10 @@
 import org.springframework.test.context.ContextConfiguration;
 import org.testng.annotations.Test;
 
-import com.evolveum.midpoint.model.impl.expr.MidpointFunctionsImpl;
-import com.evolveum.midpoint.repo.sqlbase.perfmon.SqlPerformanceMonitorImpl;
-import com.evolveum.midpoint.report.impl.ReportServiceImpl;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.task.api.Task;
-import com.evolveum.midpoint.test.TestResource;
+import com.evolveum.midpoint.test.TestReport;
+import com.evolveum.midpoint.test.TestTask;
 import com.evolveum.midpoint.util.MiscUtil;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
 
@@ -28,9 +26,12 @@ public class TestCsvReportAllAssignments extends TestCsvReport {
 
     private static final int USERS = 50;
 
-    static final TestResource<ReportType> REPORT_INDIRECT_ASSIGNMENTS = new TestResource<>(TEST_DIR_REPORTS,
+    static final TestReport REPORT_INDIRECT_ASSIGNMENTS = TestReport.classPath(DIR_REPORTS,
             "report-indirect-assignments.xml", "7f1695f2-d826-4d78-a046-b8249b79d2b5");
 
+    static final TestTask TASK_EXPORT_CLASSIC_ROLE_CACHING = new TestTask(TEST_DIR_REPORTS,
+            "task-export-role-caching.xml", "0ff414b6-76c6-4d38-95e8-d2d34c7a11cb");
+
     @Override
     public void initSystem(Task initTask, OperationResult initResult) throws Exception {
         // Only for Native repo, as Generic repo does not support reference search.
@@ -39,10 +40,10 @@ public void initSystem(Task initTask, OperationResult initResult) throws Excepti
         }
         super.initSystem(initTask, initResult);
 
-        repoAdd(TASK_EXPORT_CLASSIC, initResult);
-
+        repoAdd(TASK_EXPORT_CLASSIC_ROLE_CACHING, initResult);
         repoAdd(REPORT_INDIRECT_ASSIGNMENTS, initResult);
 
+        // adding role chain: businessRole -> appRole -> appService
         String appServiceOid = addObject(new ServiceType().name("appService").asPrismObject(), initTask, initResult);
         String appRoleOid = addObject(new RoleType().name("appRole")
                 .inducement(new AssignmentType().targetRef(appServiceOid, ServiceType.COMPLEX_TYPE))
@@ -78,8 +79,9 @@ public void initSystem(Task initTask, OperationResult initResult) throws Excepti
     @Test
     public void test100RunReport() throws Exception {
         skipIfNotNativeRepository();
+
         // 50 * 3 (normal) + 50 // 3 (direct assignments) + 3 (without metadata) + jack + header
         // (subscription footer is considered automatically later, do not count it here)
-        testClassicExport(REPORT_INDIRECT_ASSIGNMENTS, 171, 7, null);
+        testExport(TASK_EXPORT_CLASSIC_ROLE_CACHING, REPORT_INDIRECT_ASSIGNMENTS, 171, 7, null, null);
     }
 }
diff --git a/model/report-impl/src/test/java/com/evolveum/midpoint/report/TestCsvReportExportClassic.java b/model/report-impl/src/test/java/com/evolveum/midpoint/report/TestCsvReportExportClassic.java
@@ -12,8 +12,6 @@
 
 import java.io.File;
 
-import com.evolveum.midpoint.schema.util.ReportParameterTypeUtil;
-
 import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.test.context.ContextConfiguration;
 import org.testng.annotations.Test;

diff --git a/model/report-impl/src/test/resources/common/system-configuration.xml b/model/report-impl/src/test/resources/common/system-configuration.xml
@@ -6,16 +6,16 @@
   ~ and European Union Public License. See LICENSE file for details.
   -->
 <systemConfiguration oid="00000000-0000-0000-0000-000000000001" version="0"
-    xmlns="http://midpoint.evolveum.com/xml/ns/public/common/common-3"
-    xmlns:c="http://midpoint.evolveum.com/xml/ns/public/common/common-3">
+        xmlns="http://midpoint.evolveum.com/xml/ns/public/common/common-3"
+        xmlns:c="http://midpoint.evolveum.com/xml/ns/public/common/common-3">
     <name>SystemConfiguration</name>
     <logging>
         <rootLoggerAppender>File Appender</rootLoggerAppender>
         <rootLoggerLevel>INFO</rootLoggerLevel>
         <classLogger>
             <level>TRACE</level>
             <package>com.evolveum.midpoint.common.LoggingConfigurationManager</package>
-          </classLogger>
+        </classLogger>
         <c:classLogger>
             <c:level>TRACE</c:level>
             <c:package>com.evolveum.midpoint.notifications</c:package>
@@ -38,4 +38,32 @@
         </handler>
     </notificationConfiguration>
 
+    <internals>
+        <caching>
+            <profile>
+                <!--
+                Profile caching role objects (any abstract role subtype) for tasks that are expected
+                to repeatedly ask for (likely) a small subset of the roles/orgs/etc.
+                See task: task-export-role-caching.xml
+                -->
+                <name>task-role-cache</name>
+                <!-- not enabled by default -->
+                <globalRepoObjectCache>
+                    <append>true</append>
+                    <timeToLive>60</timeToLive>
+                    <maxSize>1000</maxSize> <!-- sanity limit -->
+                    <statistics>
+                        <classification>perCacheAndObjectType</classification>
+                    </statistics>
+                    <objectTypeSettings>
+                        <objectType>ArchetypeType</objectType> <!-- normally in default as well, that's OK -->
+                        <objectType>OrgType</objectType>
+                        <objectType>RoleType</objectType>
+                        <objectType>ServiceType</objectType>
+                    </objectTypeSettings>
+                </globalRepoObjectCache>
+            </profile>
+        </caching>
+    </internals>
+
 </systemConfiguration>