Skip to content

Commit

Permalink
ReportDataCreationActivityRun: fixed lower bound for audit bucketing
Browse files Browse the repository at this point in the history 
Query for lowest boundary didn't use order by timestamp.
Manual backport from master commit 323130f
  • Loading branch information
@virgo47
virgo47 committed Mar 17, 2022
1 parent d126a3f commit ca75f56
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 70 deletions.
75 changes: 44 additions & 31 deletions ...c/main/java/com/evolveum/midpoint/report/impl/activity/ReportDataCreationActivityRun.java
View file
Original file line number Diff line number Diff line change
@@ -1,49 +1,47 @@
/*
* Copyright (C) 2010-2021 Evolveum and contributors
* Copyright (C) 2010-2022 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/

package com.evolveum.midpoint.report.impl.activity;

import static com.evolveum.midpoint.report.impl.ReportUtils.getDirection;
import static com.evolveum.midpoint.util.MiscUtil.*;
import static com.evolveum.midpoint.util.MiscUtil.stateCheck;
import static com.evolveum.midpoint.xml.ns._public.common.common_3.DirectionTypeType.EXPORT;

import java.util.Collection;
import java.util.List;
import java.util.Objects;
import javax.xml.datatype.XMLGregorianCalendar;

import org.jetbrains.annotations.NotNull;

import com.evolveum.midpoint.prism.Containerable;
import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.PrismValue;
import com.evolveum.midpoint.prism.PrismPropertyValue;
import com.evolveum.midpoint.prism.query.*;
import com.evolveum.midpoint.repo.common.activity.run.ActivityRunException;
import com.evolveum.midpoint.repo.common.activity.run.ActivityRunInstantiationContext;
import com.evolveum.midpoint.repo.common.activity.run.processing.ItemProcessingRequest;
import com.evolveum.midpoint.repo.common.activity.run.SearchBasedActivityRun;
import com.evolveum.midpoint.repo.common.activity.run.SearchSpecification;
import com.evolveum.midpoint.report.impl.controller.*;

import com.evolveum.midpoint.schema.ObjectHandler;
import com.evolveum.midpoint.schema.SearchResultList;
import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.*;

import org.jetbrains.annotations.NotNull;

import com.evolveum.midpoint.repo.common.activity.run.processing.ItemProcessingRequest;
import com.evolveum.midpoint.report.impl.ReportServiceImpl;
import com.evolveum.midpoint.report.impl.ReportUtils;
import com.evolveum.midpoint.report.impl.controller.*;
import com.evolveum.midpoint.schema.GetOperationOptions;
import com.evolveum.midpoint.schema.ObjectHandler;
import com.evolveum.midpoint.schema.SearchResultList;
import com.evolveum.midpoint.schema.SelectorOptions;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.task.api.RunningTask;
import com.evolveum.midpoint.util.exception.*;
import com.evolveum.midpoint.util.exception.CommonException;
import com.evolveum.midpoint.util.exception.SchemaException;
import com.evolveum.midpoint.util.exception.SecurityViolationException;
import com.evolveum.midpoint.util.logging.Trace;
import com.evolveum.midpoint.util.logging.TraceManager;

import javax.xml.datatype.XMLGregorianCalendar;
import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.*;

/**
* Executes parts of distributed report data creation activity:
Expand Down Expand Up @@ -144,16 +142,20 @@ private void initializeAuditReportBucketing(OperationResult result)
XMLGregorianCalendar reportFrom = null;
XMLGregorianCalendar reportTo = null;
if (filter != null) {
reportTo = getTimestampFromFilter(filter, false);
reportFrom = getTimestampFromFilter(filter, true);
reportTo = getTimestampFromFilter(filter, TimestampCondition.LESS);
reportFrom = getTimestampFromFilter(filter, TimestampCondition.GREATER);
}

XMLGregorianCalendar reportToRealizedTime = Objects.requireNonNull(
activityState.getRealizationStartTimestamp(),
"no realization start timestamp for " + this);

ObjectQuery query = PrismContext.get().queryFor(AuditEventRecordType.class).build();
query.setPaging(PrismContext.get().queryFactory().createPaging(0, 1));

ObjectPaging paging = PrismContext.get().queryFactory().createPaging(0, 1);
paging.setOrdering(AuditEventRecordType.F_TIMESTAMP, OrderDirection.ASCENDING);
query.setPaging(paging);

@NotNull SearchResultList<AuditEventRecordType> firstAudit = reportService.getAuditService().searchObjects(query, null, result);
XMLGregorianCalendar reportFromFirstAuditRecords;
if (firstAudit.size() == 1) {
Expand All @@ -175,8 +177,9 @@ private void initializeAuditReportBucketing(OperationResult result)
reportFrom, reportTo, reportToRealizedTime, reportFromFirstAuditRecords);
}

private static XMLGregorianCalendar getTimestampFromFilter(ObjectFilter filter, boolean greaterOrLess) throws SchemaException {
Collection<PrismValue> values = getTimestampsFromFilter(filter, greaterOrLess);
private static XMLGregorianCalendar getTimestampFromFilter(
ObjectFilter filter, TimestampCondition timestampCondition) throws SchemaException {
Collection<PrismPropertyValue<XMLGregorianCalendar>> values = getTimestampsFromFilter(filter, timestampCondition);
if (values == null || values.size() == 0) {
return null;
} else if (values.size() > 1) {
Expand All @@ -186,25 +189,31 @@ private static XMLGregorianCalendar getTimestampFromFilter(ObjectFilter filter,
}
}

private static <T extends PrismValue> Collection<T> getTimestampsFromFilter(ObjectFilter filter, boolean greaterOrLess) {
if (greaterOrLess && filter instanceof GreaterFilter
private static Collection<PrismPropertyValue<XMLGregorianCalendar>> getTimestampsFromFilter(
ObjectFilter filter, TimestampCondition timestampCondition) {
if (timestampCondition == TimestampCondition.GREATER
&& filter instanceof GreaterFilter
&& AuditEventRecordType.F_TIMESTAMP.equivalent(((GreaterFilter<?>) filter).getFullPath())) {
return ((GreaterFilter) filter).getValues();
} else if (!greaterOrLess && filter instanceof LessFilter
//noinspection unchecked
return ((GreaterFilter<XMLGregorianCalendar>) filter).getValues();
} else if (timestampCondition == TimestampCondition.LESS
&& filter instanceof LessFilter
&& AuditEventRecordType.F_TIMESTAMP.equivalent(((LessFilter<?>) filter).getFullPath())) {
return ((LessFilter) filter).getValues();
//noinspection unchecked
return ((LessFilter<XMLGregorianCalendar>) filter).getValues();
} else if (filter instanceof AndFilter || filter instanceof OrFilter) {
return getTimestampsFromFilter(((NaryLogicalFilter) filter).getConditions(), greaterOrLess);
return getTimestampsFromFilter(((NaryLogicalFilter) filter).getConditions(), timestampCondition);
} else if (filter instanceof TypeFilter) {
return getTimestampsFromFilter(((TypeFilter) filter).getFilter(), greaterOrLess);
return getTimestampsFromFilter(((TypeFilter) filter).getFilter(), timestampCondition);
} else {
return null;
}
}

private static <T extends PrismValue> Collection<T> getTimestampsFromFilter(List<? extends ObjectFilter> conditions, boolean greaterOrLess) {
private static Collection<PrismPropertyValue<XMLGregorianCalendar>> getTimestampsFromFilter(
List<? extends ObjectFilter> conditions, TimestampCondition timestampCondition) {
for (ObjectFilter f : conditions) {
Collection<T> values = getTimestampsFromFilter(f, greaterOrLess);
Collection<PrismPropertyValue<XMLGregorianCalendar>> values = getTimestampsFromFilter(f, timestampCondition);
if (values != null) {
return values;
}
Expand Down Expand Up @@ -263,4 +272,8 @@ public void run(ObjectHandler<Containerable> handler, OperationResult result) {
// no-op
}
}

private enum TimestampCondition {
GREATER, LESS
}
}
62 changes: 23 additions & 39 deletions model/report-impl/src/test/java/com/evolveum/midpoint/report/TestCsvReportMultiNode.java
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,30 @@
/*
* Copyright (c) 2010-2019 Evolveum and contributors
* Copyright (C) 2010-2022 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/
package com.evolveum.midpoint.report;

import static org.testng.Assert.assertTrue;

import java.io.File;
import java.util.List;

import org.testng.annotations.Test;

import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.prism.path.ItemName;
import com.evolveum.midpoint.prism.path.ItemPath;
import com.evolveum.midpoint.repo.sqale.SqaleRepositoryService;
import com.evolveum.midpoint.util.exception.*;

import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType;
import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventStageType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.*;

import com.evolveum.prism.xml.ns._public.query_3.SearchFilterType;

import org.testng.SkipException;
import org.testng.annotations.Test;

import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.task.api.Task;
import com.evolveum.midpoint.test.TestResource;

import static org.testng.AssertJUnit.assertTrue;
import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectCollectionType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ReportType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.TaskType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.WorkDefinitionsType;
import com.evolveum.prism.xml.ns._public.query_3.SearchFilterType;

public class TestCsvReportMultiNode extends TestCsvReport {

Expand Down Expand Up @@ -85,71 +80,60 @@ public void test100ExportUsers() throws Exception {

@Test
public void test101ExportAuditRecords() throws Exception {
checkSqaleRepo();
auditTest();

PrismObject<ReportType> report = getObject(ReportType.class, REPORT_AUDIT_COLLECTION_WITH_DEFAULT_COLUMN.oid);
List<String> rows = basicCheckOutputFile(report, -1, 8, null);
assertTrue("Unexpected number of rows in report. Expected:1000-1010, Actual:" + rows.size(), rows.size() > 1000 && rows.size() <= 10010);
assertTrue(rows.size() > 1000 && rows.size() <= 1010,
"Unexpected number of rows in report. Expected:1000-1010, Actual:" + rows.size());
}

@Test
public void test102ExportAuditRecordsInsideTwoTimestamps() throws Exception {
checkSqaleRepo();

List<AuditEventRecordType> auditRecords = getAllAuditRecords(getTestTask(), getTestTask().getResult());
SearchFilterType filter = PrismContext.get().getQueryConverter().createSearchFilterType(
PrismContext.get().queryFor(AuditEventRecordType.class)
.item(AuditEventRecordType.F_TIMESTAMP).ge(auditRecords.get(500).getTimestamp()).and()
.item(AuditEventRecordType.F_TIMESTAMP).le(auditRecords.get(1300).getTimestamp()).buildFilter()
);
.item(AuditEventRecordType.F_TIMESTAMP).le(auditRecords.get(1300).getTimestamp()).buildFilter());

modifyObjectReplaceProperty(
ObjectCollectionType.class,
OBJECT_COLLECTION_ALL_AUDIT_RECORDS.oid,
ObjectCollectionType.F_FILTER,
getTestTask(),
getTestTask().getResult(),
filter
);
filter);

auditTest();

PrismObject<ReportType> report = getObject(ReportType.class, REPORT_AUDIT_COLLECTION_WITH_DEFAULT_COLUMN.oid);
List<String> rows = basicCheckOutputFile(report, -1, 8, null);
assertTrue("Unexpected number of rows in report. Expected:800-810, Actual:" + rows.size(), rows.size() > 800 && rows.size() <= 810);
assertTrue(rows.size() > 800 && rows.size() <= 810,
"Unexpected number of rows in report. Expected:800-810, Actual:" + rows.size());
}

@Test
public void test103ExportAuditRecordsOutsideTwoTimestamps() throws Exception {
checkSqaleRepo();

List<AuditEventRecordType> auditRecords = getAllAuditRecords(getTestTask(), getTestTask().getResult());
SearchFilterType filter = PrismContext.get().getQueryConverter().createSearchFilterType(
PrismContext.get().queryFor(AuditEventRecordType.class)
.item(AuditEventRecordType.F_TIMESTAMP).ge(auditRecords.get(1300).getTimestamp()).or()
.item(AuditEventRecordType.F_TIMESTAMP).le(auditRecords.get(500).getTimestamp()).buildFilter()
);
.item(AuditEventRecordType.F_TIMESTAMP).le(auditRecords.get(500).getTimestamp()).buildFilter());

modifyObjectReplaceProperty(
ObjectCollectionType.class,
OBJECT_COLLECTION_ALL_AUDIT_RECORDS.oid,
ObjectCollectionType.F_FILTER,
getTestTask(),
getTestTask().getResult(),
filter
);

filter);

auditTest();

PrismObject<ReportType> report = getObject(ReportType.class, REPORT_AUDIT_COLLECTION_WITH_DEFAULT_COLUMN.oid);
List<String> rows = basicCheckOutputFile(report, -1, 8, null);
assertTrue("Unexpected number of rows in report. Expected:1200-1250, Actual:" + rows.size(), rows.size() > 1200 && rows.size() <= 1250);
}

private void checkSqaleRepo() {
if (!(plainRepositoryService instanceof SqaleRepositoryService)) {
throw new SkipException("Skipping test before it is relevant only for sqale repo");
}
assertTrue(rows.size() > 1200 && rows.size() <= 1250,
"Unexpected number of rows in report. Expected:1200-1250, Actual:" + rows.size());
}

private void auditTest() throws Exception{
Expand Down

0 comments on commit ca75f56

Please sign in to comment.