Fix handling reportDataRef in distributed reports

Distributed reports created two ReportDataOutput objects, by mistake.
One at the beginning of the execution, to serve as a parent for partial
data objects, and another one at the end, to hold the aggregated data.

This is now fixed. At the end, we update the pre-existing one with the
final information.

This may be a slightly incompatible change, as it requires "MODIFY"
authorizations in addition to "ADD" ones in order to run distributed
report exports.

Plus some cosmetic/readability improvements.

model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/activity/ClassicCollectionReportExportActivityRun.java

@@ -44,8 +44,7 @@ public final class ClassicCollectionReportExportActivityRun<T>
     private ReportDataWriter<ExportedReportDataRow, ExportedReportHeaderRow> dataWriter;
 
     /**
-     * Execution object (~ controller) that is used to transfer objects found into report data.
-     * Initialized on the activity execution start.
+     * Translates objects found into report data. Initialized on the activity run start.
      */
     private CollectionExportController<T> controller;
 
@@ -73,7 +72,7 @@ public final class ClassicCollectionReportExportActivityRun<T>
     @Override
     public void beforeRun(OperationResult result) throws ActivityRunException, CommonException {
         RunningTask task = getRunningTask();
-        support.beforeExecution(result);
+        support.beforeRun(result);
         @NotNull ReportType report = support.getReport();
 
         support.stateCheck(result);
@@ -123,17 +122,16 @@ public void iterateOverItemsInBucket(OperationResult gResult) throws CommonExcep
     }
 
     @Override
-    public boolean processItem(@NotNull ItemProcessingRequest<T> request, @NotNull RunningTask workerTask,
-            OperationResult result)
-            throws CommonException, ActivityRunException {
+    public boolean processItem(
+            @NotNull ItemProcessingRequest<T> request, @NotNull RunningTask workerTask, OperationResult result) {
         T record = request.getItem();
         controller.handleDataRecord(request.getSequentialNumber(), record, workerTask, result);
         return true;
     }
 
     @Override
-    public void afterRun(OperationResult result) throws CommonException, ActivityRunException {
-        support.saveReportFile(dataWriter, result);
+    public void afterRun(OperationResult result) throws CommonException {
+        support.saveSimpleReportData(dataWriter, result);
     }
 
     @Override

model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/activity/ClassicDashboardReportExportActivityRun.java

@@ -76,7 +76,7 @@ public final class ClassicDashboardReportExportActivityRun
     @Override
     public void beforeRun(OperationResult result) throws ActivityRunException, CommonException {
         RunningTask task = getRunningTask();
-        support.beforeExecution(result);
+        support.beforeRun(result);
         @NotNull ReportType report = support.getReport();
 
         support.stateCheck(result);
@@ -174,9 +174,10 @@ public void iterateOverItemsInBucket(OperationResult gResult) throws CommonExcep
     }
 
     @Override
-    public boolean processItem(@NotNull ItemProcessingRequest<ExportDashboardReportLine<Containerable>> request,
+    public boolean processItem(
+            @NotNull ItemProcessingRequest<ExportDashboardReportLine<Containerable>> request,
             @NotNull RunningTask workerTask, OperationResult result)
-            throws CommonException, ActivityRunException {
+            throws CommonException {
 
         ExportDashboardReportLine<Containerable> item = request.getItem();
         getController(item)
@@ -189,14 +190,14 @@ private <C extends Containerable> ExportController<C> getController(@NotNull Exp
             //noinspection unchecked
             return (ExportController<C>) basicWidgetController;
         }
-        DashboardWidgetHolder holder = mapOfWidgetsController.get(item.getWidgetIdentifier());
+        DashboardWidgetHolder<?> holder = mapOfWidgetsController.get(item.getWidgetIdentifier());
         //noinspection unchecked
         return (ExportController<C>) holder.getController();
     }
 
     @Override
     public void afterRun(OperationResult result) throws CommonException {
-        support.saveReportFile(dataWriter, result);
+        support.saveSimpleReportData(dataWriter, result);
     }
 
     @Override

model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/activity/ClassicReportImportActivityRun.java

@@ -7,33 +7,27 @@
 
 package com.evolveum.midpoint.report.impl.activity;
 
+import static com.evolveum.midpoint.schema.result.OperationResultStatus.FATAL_ERROR;
+import static com.evolveum.midpoint.task.api.TaskRunResult.TaskRunResultStatus.PERMANENT_ERROR;
+
 import java.io.IOException;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BiConsumer;
+
+import org.jetbrains.annotations.NotNull;
 
 import com.evolveum.midpoint.repo.common.activity.run.*;
 import com.evolveum.midpoint.repo.common.activity.run.processing.ItemProcessingRequest;
 import com.evolveum.midpoint.report.impl.ReportServiceImpl;
-
 import com.evolveum.midpoint.report.impl.controller.ImportController;
 import com.evolveum.midpoint.schema.expression.VariablesMap;
-
-import com.evolveum.midpoint.task.api.RunningTask;
-import com.evolveum.midpoint.util.logging.Trace;
-import com.evolveum.midpoint.util.logging.TraceManager;
-
-import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
-
-import org.jetbrains.annotations.NotNull;
-
-import com.evolveum.midpoint.repo.common.activity.run.ActivityRunException;
 import com.evolveum.midpoint.schema.result.OperationResult;
+import com.evolveum.midpoint.task.api.RunningTask;
 import com.evolveum.midpoint.util.exception.CommonException;
-
-import static com.evolveum.midpoint.schema.result.OperationResultStatus.FATAL_ERROR;
-import static com.evolveum.midpoint.task.api.TaskRunResult.TaskRunResultStatus.PERMANENT_ERROR;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.AbstractActivityWorkStateType;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ActivityOverallItemCountingOptionType;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ReportType;
 
 /**
  * Activity execution for report import.
@@ -45,8 +39,6 @@ final class ClassicReportImportActivityRun
                 ClassicReportImportActivityHandler,
                 AbstractActivityWorkStateType> {
 
-    private static final Trace LOGGER = TraceManager.getTrace(ClassicReportImportActivityRun.class);
-
     @NotNull private final ImportActivitySupport support;
 
     /** The report service Spring bean. */
@@ -73,7 +65,7 @@ final class ClassicReportImportActivityRun
 
     @Override
     public void beforeRun(OperationResult result) throws CommonException, ActivityRunException {
-        support.beforeExecution(result);
+        support.beforeRun(result);
         ReportType report = support.getReport();
 
         support.stateCheck(result);
@@ -96,7 +88,7 @@ public void beforeRun(OperationResult result) throws CommonException, ActivityRu
     }
 
     @Override
-    public Integer determineOverallSize(OperationResult result) throws CommonException {
+    public Integer determineOverallSize(OperationResult result) {
         return variables.size();
     }
 
@@ -116,9 +108,9 @@ public void iterateOverItemsInBucket(OperationResult result) {
     }
 
     @Override
-    public boolean processItem(@NotNull ItemProcessingRequest<InputReportLine> request, @NotNull RunningTask workerTask,
-            OperationResult result)
-            throws CommonException, ActivityRunException {
+    public boolean processItem(
+            @NotNull ItemProcessingRequest<InputReportLine> request, @NotNull RunningTask workerTask, OperationResult result)
+            throws CommonException {
         InputReportLine line = request.getItem();
         controller.handleDataRecord(line, workerTask, result);
         return true;

model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/activity/DistributedReportExportActivityHandler.java

@@ -47,6 +47,12 @@
  *
  * 1. Partial reports creation: report data is created for each bucket of objects.
  * 2. Report summarization: partial report data objects are aggregated into summary one.
+ *
+ * State maintained:
+ *
+ * . `reportDataRef` in the parent activity state -> this is the ultimate aggregated (summary) report that contains data
+ * from partial data reports; created at the beginning and updated at the end
+ * . for compatibility reasons, `reportDataRef` is stored also in the aggregation sub-activity and in task extension
  */
 @Component
 public class DistributedReportExportActivityHandler
@@ -63,13 +69,15 @@ public class DistributedReportExportActivityHandler
 
     @PostConstruct
     public void register() {
-        registry.register(DistributedReportExportWorkDefinitionType.COMPLEX_TYPE, null,
+        registry.register(
+                DistributedReportExportWorkDefinitionType.COMPLEX_TYPE, null,
                 DistributedReportExportWorkDefinition.class, DistributedReportExportWorkDefinition::new, this);
     }
 
     @PreDestroy
     public void unregister() {
-        registry.unregister(DistributedReportExportWorkDefinitionType.COMPLEX_TYPE, null,
+        registry.unregister(
+                DistributedReportExportWorkDefinitionType.COMPLEX_TYPE, null,
                 DistributedReportExportWorkDefinition.class);
     }
 
@@ -107,16 +115,19 @@ public void unregister() {
         return children;
     }
 
+    /**
+     * This should be the ultimate (aggregated) report data object. It is created at the very beginning.
+     *
+     * It would seem to be useless to create such object at the beginning, as it is basically overwritten during aggregation
+     * sub-activity run. But its OID is used as both `parentRef` as well as a part of the name for partial report data objects,
+     * binding them together.
+     */
     private void createEmptyAggregatedDataObject(
             EmbeddedActivity<DistributedReportExportWorkDefinition, DistributedReportExportActivityHandler> activity,
             RunningTask runningTask, OperationResult result) throws CommonException {
         ActivityState activityState =
-                ActivityState.getActivityStateUpwards(
-                        activity.getPath().allExceptLast(),
-                        runningTask,
-                        ReportExportWorkStateType.COMPLEX_TYPE,
-                        commonTaskBeans,
-                        result);
+                DistributedReportExportActivitySupport.getWholeActivityState(
+                        activity.getPath().allExceptLast(), runningTask, result);
         if (activityState.getWorkStateReferenceRealValue(F_REPORT_DATA_REF) != null) {
             return;
         }

model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/activity/DistributedReportExportActivitySupport.java

@@ -13,8 +13,13 @@
 import static com.evolveum.midpoint.task.api.TaskRunResult.TaskRunResultStatus.PERMANENT_ERROR;
 import static com.evolveum.midpoint.xml.ns._public.common.common_3.ReportExportWorkStateType.F_REPORT_DATA_REF;
 
+import com.evolveum.midpoint.repo.common.activity.EmbeddedActivity;
 import com.evolveum.midpoint.repo.common.activity.run.AbstractActivityRun;
 
+import com.evolveum.midpoint.repo.common.activity.run.CommonTaskBeans;
+import com.evolveum.midpoint.schema.util.task.ActivityPath;
+import com.evolveum.midpoint.task.api.RunningTask;
+
 import org.jetbrains.annotations.NotNull;
 
 import com.evolveum.midpoint.repo.common.activity.Activity;
@@ -35,7 +40,9 @@ class DistributedReportExportActivitySupport extends ExportActivitySupport {
     @NotNull private final Activity<DistributedReportExportWorkDefinition, DistributedReportExportActivityHandler> activity;
 
     /**
-     * Global report data - point of aggregation.
+     * Global report data - point of aggregation. It is created on the very start of the {@link ReportDataCreationActivityRun},
+     * see {@link DistributedReportExportActivityHandler#createEmptyAggregatedDataObject(EmbeddedActivity, RunningTask,
+     * OperationResult)}.
      */
     private ObjectReferenceType globalReportDataRef;
 
@@ -48,24 +55,17 @@ class DistributedReportExportActivitySupport extends ExportActivitySupport {
         this.activity = activity;
     }
 
-    void beforeExecution(OperationResult result) throws CommonException, ActivityRunException {
-        super.beforeExecution(result);
+    void beforeRun(OperationResult result) throws CommonException, ActivityRunException {
+        super.beforeRun(result);
         globalReportDataRef = fetchGlobalReportDataRef(result);
     }
 
     private @NotNull ObjectReferenceType fetchGlobalReportDataRef(OperationResult result)
             throws SchemaException, ObjectNotFoundException, ActivityRunException {
-        ActivityState activityState =
-                ActivityState.getActivityStateUpwards(
-                        activity.getPath().allExceptLast(),
-                        runningTask,
-                        ReportExportWorkStateType.COMPLEX_TYPE,
-                        beans,
-                        result);
+        ActivityState activityState = getWholeActivityState(activity.getPath().allExceptLast(), runningTask, result);
         ObjectReferenceType globalReportDataRef = activityState.getWorkStateReferenceRealValue(F_REPORT_DATA_REF);
         if (globalReportDataRef == null) {
-            throw new ActivityRunException("No global report data reference in " + activityState,
-                    FATAL_ERROR, PERMANENT_ERROR);
+            throw new ActivityRunException("No global report data reference in " + activityState, FATAL_ERROR, PERMANENT_ERROR);
         }
         return globalReportDataRef;
     }
@@ -82,4 +82,21 @@ public void stateCheck(OperationResult result) throws CommonException {
         MiscUtil.stateCheck(report.getObjectCollection() != null, "Only collection-based reports are supported here");
         super.stateCheck(result);
     }
+
+    /**
+     * Returns the activity state of the whole "distributed export" activity, i.e. parent of both data creation
+     * and aggregation sub-activities.
+     *
+     * @param wholeActivityPath Path to the whole (parent) "distributed export" activity
+     */
+    static @NotNull ActivityState getWholeActivityState(
+            ActivityPath wholeActivityPath, RunningTask runningTask, OperationResult result)
+            throws SchemaException, ObjectNotFoundException {
+        return ActivityState.getActivityStateUpwards(
+                wholeActivityPath,
+                runningTask,
+                ReportExportWorkStateType.COMPLEX_TYPE,
+                CommonTaskBeans.get(),
+                result);
+    }
 }

model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/activity/ExportActivitySupport.java

@@ -31,6 +31,10 @@
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
 
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ReportDataType;
+
+import org.jetbrains.annotations.NotNull;
+
 /**
  * Contains common functionality for executions of export report-related activities.
  * This is an experiment - using object composition instead of inheritance.
@@ -44,8 +48,8 @@ public class ExportActivitySupport extends ReportActivitySupport {
         super(activityRun, reportService, resolver, workDefinition);
     }
 
-    void beforeExecution(OperationResult result) throws CommonException, ActivityRunException {
-        super.beforeExecution(result);
+    void beforeRun(OperationResult result) throws CommonException, ActivityRunException {
+        super.beforeRun(result);
         setupSaveSupport();
     }
 
@@ -54,17 +58,24 @@ private void setupSaveSupport() {
     }
 
     /**
-     * Save exported report to a file.
+     * Saves the data in the simple case: just dumping content of `dataWriter` to the output file.
      */
-    void saveReportFile(String aggregatedData,
+    void saveSimpleReportData(
             ReportDataWriter<? extends ExportedReportDataRow, ? extends ExportedReportHeaderRow> dataWriter,
             OperationResult result) throws CommonException {
-        saveSupport.saveReportFile(aggregatedData, dataWriter, result);
+        saveSupport.saveSimpleReportData(dataWriter, result);
     }
 
-    void saveReportFile(ReportDataWriter<? extends ExportedReportDataRow, ? extends ExportedReportHeaderRow> dataWriter,
-            OperationResult result) throws CommonException {
-        saveSupport.saveReportFile(dataWriter, result);
+    /**
+     * Save exported report to a file. This is the variant for distributed reports that assumes we have the
+     * aggregated data as a String, plus pre-existing (empty) aggregated {@link ReportDataType} object.
+     */
+    void saveAggregatedReportData(
+            @NotNull String aggregatedData,
+            @NotNull ReportDataWriter<? extends ExportedReportDataRow, ? extends ExportedReportHeaderRow> completingDataWriter,
+            @NotNull ObjectReferenceType aggregatedDataRef,
+            @NotNull OperationResult result) throws CommonException {
+        saveSupport.saveAggregatedReportData(aggregatedData, completingDataWriter, aggregatedDataRef, result);
     }
 
     /**
@@ -115,7 +126,7 @@ public <T> void searchRecordsIteratively(
      * Count container objects for iterative task.
      * Temporary until will be implemented iterative search for audit records and containerable objects.
      */
-    public int countRecords(Class<?> type,
+    int countRecords(Class<?> type,
             ObjectQuery query,
             Collection<SelectorOptions<GetOperationOptions>> options,
             OperationResult result) throws CommonException {

model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/activity/ExportDashboardActivitySupport.java

@@ -49,8 +49,8 @@ class ExportDashboardActivitySupport extends ExportActivitySupport {
     }
 
     @Override
-    void beforeExecution(OperationResult result) throws CommonException, ActivityRunException {
-        super.beforeExecution(result);
+    void beforeRun(OperationResult result) throws CommonException, ActivityRunException {
+        super.beforeRun(result);
         setupDashboard(result);
         setupCompiledViewsForWidgets(result);
     }
@@ -60,7 +60,8 @@ private void setupCompiledViewsForWidgets(OperationResult result) throws CommonE
         List<DashboardWidgetType> widgets = dashboard.getWidget();
         for (DashboardWidgetType widget : widgets) {
             if (isWidgetTableVisible()) {
-                CompiledObjectCollectionView compiledView = reportService.createCompiledView(report.getDashboard(), widget, runningTask, result);
+                CompiledObjectCollectionView compiledView =
+                        reportService.createCompiledView(report.getDashboard(), widget, runningTask, result);
                 DisplayType newDisplay = widget.getDisplay();
                 if (compiledView.getDisplay() == null) {
                     compiledView.setDisplay(newDisplay);