Skip to content

Commit

Permalink
implementation of implicit segmentacion for audit reports
Browse files Browse the repository at this point in the history
  • Loading branch information
@skublik
skublik committed Sep 30, 2021
1 parent ff58137 commit a89e7ee
Show file tree
Hide file tree
Showing 9 changed files with 276 additions and 56 deletions.
6 changes: 6 additions & 0 deletions model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/ReportServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.util.*;
import javax.xml.namespace.QName;

import com.evolveum.midpoint.audit.api.AuditService;
import com.evolveum.midpoint.common.Clock;
import com.evolveum.midpoint.common.LocalizationService;
import com.evolveum.midpoint.model.api.*;
Expand Down Expand Up @@ -69,6 +70,7 @@ public class ReportServiceImpl implements ReportService {
@Autowired private ExpressionFactory expressionFactory;
@Autowired @Qualifier("modelObjectResolver") private ObjectResolver objectResolver;
@Autowired @Qualifier("cacheRepositoryService") private RepositoryService repositoryService;
@Autowired private AuditService auditService;
@Autowired private ModelAuditService modelAuditService;
@Autowired private FunctionLibrary logFunctionLibrary;
@Autowired private FunctionLibrary basicFunctionLibrary;
Expand Down Expand Up @@ -348,6 +350,10 @@ public ModelAuditService getModelAuditService() {
return modelAuditService;
}

public AuditService getAuditService() {
return auditService;
}

public ModelInteractionService getModelInteractionService() {
return modelInteractionService;
}
Expand Down
121 changes: 96 additions & 25 deletions ...mpl/src/main/java/com/evolveum/midpoint/report/impl/activity/AuditReportSegmentation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,13 @@

package com.evolveum.midpoint.report.impl.activity;

import com.evolveum.midpoint.audit.api.AuditService;
import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.query.ObjectFilter;
import com.evolveum.midpoint.prism.query.ObjectQuery;
import com.evolveum.midpoint.prism.xml.XmlTypeConverter;
import com.evolveum.midpoint.report.api.ReportService;
import com.evolveum.midpoint.schema.SearchResultList;
import com.evolveum.midpoint.util.exception.SchemaException;
import com.evolveum.midpoint.util.exception.SystemException;
import com.evolveum.midpoint.util.logging.Trace;
Expand Down Expand Up @@ -37,14 +41,22 @@ class AuditReportSegmentation {

private static final Trace LOGGER = TraceManager.getTrace(AuditReportSegmentation.class);

@NotNull private final XMLGregorianCalendar reportFrom;
@NotNull private final XMLGregorianCalendar reportTo;
private final XMLGregorianCalendar reportFromFromFilter;
private final XMLGregorianCalendar reportToFromFilter;
@NotNull private final XMLGregorianCalendar reportToRealizedTime;
@NotNull private final XMLGregorianCalendar reportFromFirstAuditRecord;

private ExplicitWorkSegmentationType explicitSegmentation;

AuditReportSegmentation(@NotNull XMLGregorianCalendar reportFrom, @NotNull XMLGregorianCalendar reportTo) {
this.reportFrom = reportFrom;
this.reportTo = reportTo;
AuditReportSegmentation(
XMLGregorianCalendar reportFrom,
XMLGregorianCalendar reportTo,
@NotNull XMLGregorianCalendar reportToRealizedTime,
@NotNull XMLGregorianCalendar reportFromFirstAuditRecord) {
this.reportFromFromFilter = reportFrom;
this.reportToFromFilter = reportTo;
this.reportToRealizedTime = reportToRealizedTime;
this.reportFromFirstAuditRecord = reportFromFirstAuditRecord;
}

/**
Expand All @@ -67,17 +79,83 @@ AbstractWorkSegmentationType resolveImplicitSegmentation(ImplicitWorkSegmentatio
argCheck(segmentation.getMatchingRule() == null, "Matching rule specification is not supported");
argCheck(segmentation.getNumberOfBuckets() != null, "Number of buckets must be specified");

long reportFromMillis = XmlTypeConverter.toMillis(reportFrom);
long reportToMillis = XmlTypeConverter.toMillis(reportTo);
long step = (reportToMillis - reportFromMillis) / segmentation.getNumberOfBuckets();

LOGGER.trace("Creating segmentation: from = {}, to = {}, step = {}",
reportFrom, reportTo, step);
XMLGregorianCalendar reportFrom = reportFromFromFilter;
XMLGregorianCalendar reportTo = reportToFromFilter;

if (reportFrom == null && reportTo == null) {
reportTo = reportToRealizedTime;
reportFrom = reportFromFirstAuditRecord;
} else if (reportFrom == null) {
reportFrom = reportFromFirstAuditRecord;
if (reportToRealizedTime.toGregorianCalendar().compareTo(reportTo.toGregorianCalendar()) < 0) {
reportTo = reportToRealizedTime;
}
} else {
reportTo = reportToRealizedTime;
if (reportFromFirstAuditRecord.toGregorianCalendar().compareTo(reportFrom.toGregorianCalendar()) > 0) {
reportFrom = reportFromFirstAuditRecord;
}
}

explicitSegmentation = new ExplicitWorkSegmentationType(PrismContext.get());
for (long bucketFromMillis = reportFromMillis; bucketFromMillis < reportToMillis; bucketFromMillis += step) {
explicitSegmentation.getContent().add(
createBucketContent(bucketFromMillis, step, reportToMillis));
int result =
reportFrom.toGregorianCalendar().compareTo(reportTo.toGregorianCalendar());

if (result < 0) {
long reportFromMillis = XmlTypeConverter.toMillis(reportFrom);
long reportToMillis = XmlTypeConverter.toMillis(reportTo);
long step = (reportToMillis - reportFromMillis) / segmentation.getNumberOfBuckets();
if (step < 0) {
step = 0;
}

LOGGER.trace("Creating segmentation: from = {}, to = {}, step = {}",
reportFrom, reportTo, step);

explicitSegmentation = new ExplicitWorkSegmentationType(PrismContext.get());
for (long bucketFromMillis = reportFromMillis; bucketFromMillis < reportToMillis; bucketFromMillis += step) {
explicitSegmentation.getContent().add(
createBucketContent(bucketFromMillis, step, reportToMillis));
}
if (reportFromMillis + segmentation.getNumberOfBuckets() * step < reportToMillis) {
explicitSegmentation.getContent().add(
createBucketContent(reportFromMillis + segmentation.getNumberOfBuckets() * step, step, reportToMillis));
}
} else {
long reportFromMillis = XmlTypeConverter.toMillis(reportFrom);
long reportToMillis = XmlTypeConverter.toMillis(reportTo);
long reportFromMillisFirstRecord = XmlTypeConverter.toMillis(reportFromFirstAuditRecord);
long reportToMillisRealizedTime = XmlTypeConverter.toMillis(reportToRealizedTime);
long step = ((reportToMillis - reportFromMillisFirstRecord) + (reportToMillisRealizedTime - reportFromMillis))
/ segmentation.getNumberOfBuckets();

explicitSegmentation = new ExplicitWorkSegmentationType(PrismContext.get());
long bucketFromMillis = reportFromMillisFirstRecord;
while (bucketFromMillis + step < reportToMillis) {
explicitSegmentation.getContent().add(
createBucketContent(bucketFromMillis, step, reportToMillis));
bucketFromMillis += step;
}
if ((bucketFromMillis + step / 2) < reportToMillis) {
explicitSegmentation.getContent().add(
createBucketContent(bucketFromMillis, step * 2, reportToMillis));
bucketFromMillis = reportFromMillis;
} else {
explicitSegmentation.getContent().add(
createBucketContent(bucketFromMillis, step, reportToMillis));
bucketFromMillis = bucketFromMillis + step;
explicitSegmentation.getContent().add(
createBucketContent(bucketFromMillis, step, reportToMillis));
long firstStep = bucketFromMillis + step - reportToMillis + step;
explicitSegmentation.getContent().add(
createBucketContent(reportFromMillis, firstStep, reportToMillisRealizedTime));
bucketFromMillis = reportFromMillis + firstStep;
}

while (bucketFromMillis + step < reportToMillis) {
explicitSegmentation.getContent().add(
createBucketContent(bucketFromMillis, step, reportToMillis));
bucketFromMillis += step;
}
}
return explicitSegmentation;
}
Expand All @@ -94,20 +172,13 @@ private AbstractWorkBucketContentType createBucketContent(long bucketFromMillis,
if (bucketFromMillis + step < reportToMillis) {
bucketTo = XmlTypeConverter.createXMLGregorianCalendar(bucketFromMillis + step);
} else {
bucketTo = null;
bucketTo = XmlTypeConverter.createXMLGregorianCalendar(reportToMillis);
}

ObjectFilter filter;
if (bucketTo != null) {
filter = PrismContext.get().queryFor(AuditEventRecordType.class)
ObjectFilter filter = PrismContext.get().queryFor(AuditEventRecordType.class)
.item(AuditEventRecordType.F_TIMESTAMP).ge(bucketFrom) // inclusive
.and().item(AuditEventRecordType.F_TIMESTAMP).lt(bucketTo) // exclusive
.and().item(AuditEventRecordType.F_TIMESTAMP).le(bucketTo) // exclusive
.buildFilter();
} else {
filter = PrismContext.get().queryFor(AuditEventRecordType.class)
.item(AuditEventRecordType.F_TIMESTAMP).ge(bucketFrom)
.buildFilter();
}
SearchFilterType filterBean;
try {
filterBean = PrismContext.get().getQueryConverter().createSearchFilterType(filter);
Expand Down
88 changes: 81 additions & 7 deletions .../java/com/evolveum/midpoint/report/impl/activity/ReportDataCreationActivityExecution.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,29 @@
import static com.evolveum.midpoint.xml.ns._public.common.common_3.DirectionTypeType.EXPORT;

import java.util.Collection;
import java.util.List;
import java.util.Objects;

import com.evolveum.midpoint.prism.Containerable;
import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.PrismValue;
import com.evolveum.midpoint.prism.path.ItemName;
import com.evolveum.midpoint.prism.path.ItemPath;
import com.evolveum.midpoint.prism.query.*;
import com.evolveum.midpoint.prism.xml.XmlTypeConverter;
import com.evolveum.midpoint.repo.common.activity.execution.ExecutionInstantiationContext;
import com.evolveum.midpoint.report.impl.controller.*;

import com.evolveum.midpoint.repo.common.task.*;

import com.evolveum.midpoint.schema.SearchResultList;
import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.*;

import com.evolveum.prism.xml.ns._public.query_3.SearchFilterType;

import org.jetbrains.annotations.NotNull;

import com.evolveum.midpoint.prism.query.ObjectQuery;
import com.evolveum.midpoint.repo.common.activity.ActivityExecutionException;
import com.evolveum.midpoint.report.impl.ReportServiceImpl;
import com.evolveum.midpoint.report.impl.ReportUtils;
Expand All @@ -40,6 +48,7 @@
import com.evolveum.midpoint.util.logging.TraceManager;

import javax.xml.datatype.XMLGregorianCalendar;
import javax.xml.namespace.QName;

/**
* Executes parts of distributed report data creation activity:
Expand Down Expand Up @@ -123,18 +132,83 @@ private void initializeController(OperationResult result) throws CommonException
stateCheck(searchSpecificationHolder.searchSpecification != null, "No search specification was provided");
masterSearchSpecification = searchSpecificationHolder.searchSpecification;

if (AuditEventRecordType.class.equals(masterSearchSpecification.getType())) {
initializeAuditReportBucketing(result);
}
// if (AuditEventRecordType.class.equals(masterSearchSpecification.getType())) {
initializeAuditReportBucketing(result);
// }
}

private void initializeAuditReportBucketing(OperationResult result)
throws SchemaException, ObjectNotFoundException {
@NotNull XMLGregorianCalendar reportFrom = XmlTypeConverter.createXMLGregorianCalendar("2020-01-01T00:00:00.000+02:00");
@NotNull XMLGregorianCalendar reportTo = Objects.requireNonNull(
ObjectFilter filter = masterSearchSpecification.getQuery().getFilter();
XMLGregorianCalendar reportFrom = null;
XMLGregorianCalendar reportTo = null;
if (filter != null) {
reportTo = getTimestampFromFilter(filter, false);
reportFrom = getTimestampFromFilter(filter, true);
}

XMLGregorianCalendar reportToRealizedTime = Objects.requireNonNull(
activityState.getRealizationStartTimestamp(),
"no realization start timestamp for " + this);
auditReportSegmentation = new AuditReportSegmentation(reportFrom, reportTo);

ObjectQuery query = PrismContext.get().queryFor(AuditEventRecordType.class).build();
query.setPaging(PrismContext.get().queryFactory().createPaging(0, 1));
@NotNull SearchResultList<AuditEventRecordType> firstAudit = reportService.getAuditService().searchObjects(query, null, result);
XMLGregorianCalendar reportFromFirstAuditRecords;
if (firstAudit.size() == 1) {
reportFromFirstAuditRecords = firstAudit.iterator().next().getTimestamp();
} else {
LOGGER.debug("Couldn't find fist audit record in repository.");
return;
}
int compareOfFirstWithLastDate =
reportFromFirstAuditRecords.toGregorianCalendar().compareTo(reportToRealizedTime.toGregorianCalendar());
if ((compareOfFirstWithLastDate >= 0)
|| (reportFrom != null && reportTo != null
&& reportFrom.toGregorianCalendar().compareTo(reportTo.toGregorianCalendar()) == 0)) {
LOGGER.debug("Couldn't report any audit records.");
return;
}

auditReportSegmentation = new AuditReportSegmentation(
reportFrom, reportTo, reportToRealizedTime, reportFromFirstAuditRecords);
}

private static XMLGregorianCalendar getTimestampFromFilter(ObjectFilter filter, boolean greaterOrLess) throws SchemaException {
Collection<PrismValue> values = getTimestampsFromFilter(filter, greaterOrLess);
if (values == null || values.size() == 0) {
return null;
} else if (values.size() > 1) {
throw new SchemaException("More than one " + AuditEventRecordType.F_TIMESTAMP + " defined in the search query.");
} else {
return values.iterator().next().getRealValue();
}
}

private static <T extends PrismValue> Collection<T> getTimestampsFromFilter(ObjectFilter filter, boolean greaterOrLess) {
if (greaterOrLess && filter instanceof GreaterFilter
&& AuditEventRecordType.F_TIMESTAMP.equivalent(((GreaterFilter) filter).getFullPath())) {
return ((GreaterFilter) filter).getValues();
} else if (!greaterOrLess && filter instanceof LessFilter
&& AuditEventRecordType.F_TIMESTAMP.equivalent(((LessFilter) filter).getFullPath())) {
return ((LessFilter) filter).getValues();
} else if (filter instanceof AndFilter) {
return getTimestampsFromFilter(((NaryLogicalFilter) filter).getConditions(), greaterOrLess);
} else if (filter instanceof TypeFilter) {
return getTimestampsFromFilter(((TypeFilter) filter).getFilter(), greaterOrLess);
} else {
return null;
}
}

private static <T extends PrismValue> Collection<T> getTimestampsFromFilter(List<? extends ObjectFilter> conditions, boolean greaterOrLess) {
for (ObjectFilter f : conditions) {
Collection<T> values = getTimestampsFromFilter(f, greaterOrLess);
if (values != null) {
return values;
}
}
return null;
}

/**
Expand Down
12 changes: 9 additions & 3 deletions model/report-impl/src/test/java/com/evolveum/midpoint/report/EmptyReportIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
import java.util.Date;
import java.util.List;

import com.evolveum.midpoint.prism.path.ItemName;

import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;

Expand Down Expand Up @@ -169,15 +171,15 @@ void commonInitialization(OperationResult initResult)
login(userAdministrator);
}

void createUsers(int users, OperationResult initResult) throws CommonException {
void createUsers(int users, Task initTask, OperationResult initResult) throws CommonException {
for (int i = 0; i < users; i++) {
UserType user = new UserType(prismContext)
.name(String.format("u%06d", i))
.givenName(String.format("GivenNameU%06d", i))
.familyName(String.format("FamilyNameU%06d", i))
.fullName(String.format("FullNameU%06d", i))
.emailAddress(String.format("EmailU%06d@test.com", i));
repositoryService.addObject(user.asPrismObject(), null, initResult);
addObject(user.asPrismObject(), initTask, initResult);
}
System.out.printf("%d users created", users);
}
Expand Down Expand Up @@ -250,12 +252,16 @@ void runExportTask(TestResource<TaskType> taskResource, TestResource<ReportType>
changeTaskReport(reportResource,
ItemPath.create(TaskType.F_ACTIVITY,
ActivityDefinitionType.F_WORK,
WorkDefinitionsType.F_REPORT_EXPORT,
getWorkDefinitionType(),
ClassicReportImportWorkDefinitionType.F_REPORT_REF
),
taskResource);
rerunTask(taskResource.oid, result);
}

protected ItemName getWorkDefinitionType() {
return WorkDefinitionsType.F_REPORT_EXPORT;
}

protected abstract FileFormatConfigurationType getFileFormatConfiguration();
}
7 changes: 3 additions & 4 deletions model/report-impl/src/test/java/com/evolveum/midpoint/report/TestCsvReport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,26 +6,25 @@
*/
package com.evolveum.midpoint.report;

import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.util.List;

import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.prism.path.ItemName;
import com.evolveum.midpoint.util.exception.*;

import com.evolveum.midpoint.xml.ns._public.common.common_3.FileFormatConfigurationType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.FileFormatTypeType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectCollectionType;

import com.evolveum.midpoint.xml.ns._public.common.common_3.WorkDefinitionsType;

import org.apache.commons.lang3.StringUtils;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.testng.annotations.Test;

import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.task.api.Task;
import com.evolveum.midpoint.test.TestResource;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ReportType;

@ContextConfiguration(locations = { "classpath:ctx-report-test-main.xml" })
Expand Down
2 changes: 1 addition & 1 deletion model/report-impl/src/test/java/com/evolveum/midpoint/report/TestCsvReportExportClassic.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public void initSystem(Task initTask, OperationResult initResult) throws Excepti
repoAdd(DASHBOARD_DEFAULT_COLUMNS, initResult);
repoAdd(DASHBOARD_EMPTY, initResult);

createUsers(USERS, initResult);
createUsers(USERS, initTask, initResult);
}

@Test
Expand Down

0 comments on commit a89e7ee

Please sign in to comment.