From 2b79ef3cd30eaf1a95bde9b5e935d3d3b73ead0a Mon Sep 17 00:00:00 2001 From: lskublik Date: Thu, 3 Sep 2020 15:07:18 +0200 Subject: [PATCH 01/12] adding of using query api for audit reports and dashboard with audit widgets instead of SQL query --- .../schema/constants/AuditConstants.java | 47 ----- .../xml/ns/public/common/audit-3.xsd | 69 +++++- .../xml/ns/public/common/common-core-3.xsd | 2 + .../api/interaction/DashboardService.java | 3 +- .../model/common/util/DefaultColumnUtils.java | 107 +++------- .../impl/controller/DashboardServiceImpl.java | 197 ++++++++++++++---- .../controller/fileformat/CsvController.java | 37 ++-- .../fileformat/FileFormatController.java | 135 +++--------- .../controller/fileformat/HtmlController.java | 32 +-- .../object-collection-all-audit-records.xml | 17 +- .../midpoint/audit/api/AuditEventRecord.java | 10 + 11 files changed, 353 insertions(+), 303 deletions(-) delete mode 100644 infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/AuditConstants.java diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/AuditConstants.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/AuditConstants.java deleted file mode 100644 index dc92bae3a04..00000000000 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/constants/AuditConstants.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2010-2019 Evolveum and contributors - * - * This work is dual-licensed under the Apache License 2.0 - * and European Union Public License. See LICENSE file for details. - */ -package com.evolveum.midpoint.schema.constants; - -/** - * Constants for localization key of COLUMN_NAMEs labels of audit widget report. - * - * @author skublik - */ -public class AuditConstants { - - public static final String TIME_COLUMN = "time"; - public static final String INITIATOR_COLUMN = "initiator"; - public static final String EVENT_STAGE_COLUMN = "eventStage"; - public static final String EVENT_TYPE_COLUMN = "eventType"; - public static final String TARGET_COLUMN = "target"; - public static final String DELTA_COLUMN = "delta"; - public static final String MESSAGE_COLUMN = "message"; - public static final String TARGET_OWNER_COLUMN = "targetOwner"; - public static final String CHANNEL_COLUMN = "channel"; - public static final String OUTCOME_COLUMN = "outcome"; - public static final String TASK_OID_COLUMN = "taskOid"; - public static final String NODE_IDENTIFIER_COLUMN = "nodeIdentifier"; - public static final String ATTORNEY_COLUMN = "attorney"; - public static final String RESULT_COLUMN = "result"; - public static final String RESOURCE_OID_COLUMN = "resourceOid"; - - public static final String TIME_COLUMN_KEY = "AuditLocalizationConstants.time"; - public static final String INITIATOR_COLUMN_KEY = "AuditLocalizationConstants.initiator"; - public static final String EVENT_STAGE_COLUMN_KEY = "AuditLocalizationConstants.eventStage"; - public static final String EVENT_TYPE_COLUMN_KEY = "AuditLocalizationConstants.eventType"; - public static final String TARGET_COLUMN_KEY = "AuditLocalizationConstants.target"; - public static final String DELTA_COLUMN_KEY = "AuditLocalizationConstants.delta"; - public static final String MESSAGE_COLUMN_KEY = "AuditLocalizationConstants.message"; - public static final String TARGET_OWNER_COLUMN_KEY = "AuditLocalizationConstants.targetOwner"; - public static final String CHANNEL_COLUMN_KEY = "AuditLocalizationConstants.channel"; - public static final String OUTCOME_COLUMN_KEY = "AuditLocalizationConstants.outcome"; - public static final String TASK_OID_COLUMN_KEY = "AuditLocalizationConstants.taskOid"; - public static final String NODE_IDENTIFIER_COLUMN_KEY = "AuditLocalizationConstants.nodeIdentifier"; - public static final String ATTORNEY_COLUMN_KEY = "AuditLocalizationConstants.attorney"; - public static final String RESULT_COLUMN_KEY = "AuditLocalizationConstants.result"; - public static final String RESOURCE_OID_COLUMN_KEY = "AuditLocalizationConstants.resourceOid"; -} diff --git a/infra/schema/src/main/resources/xml/ns/public/common/audit-3.xsd b/infra/schema/src/main/resources/xml/ns/public/common/audit-3.xsd index cc89ec9588c..8ab87d1b5ed 100644 --- a/infra/schema/src/main/resources/xml/ns/public/common/audit-3.xsd +++ b/infra/schema/src/main/resources/xml/ns/public/common/audit-3.xsd @@ -77,6 +77,9 @@ Timestamp when the event occurred. + + AuditEventRecordType.timestamp + @@ -85,6 +88,9 @@ Unique identification of the event. Every record should have unique event identifier. + + AuditEventRecordType.eventIdentifier + @@ -92,6 +98,9 @@ Identification of (interactive) session in which the event occurred. + + AuditEventRecordType.sessionIdentifier + @@ -112,6 +121,9 @@ operation will have a separate request identifier.

+ + AuditEventRecordType.requestIdentifier + @@ -133,6 +145,9 @@ operation will have a separate request identifier.

+ + AuditEventRecordType.taskIdentifier + @@ -141,6 +156,9 @@ Task OID. This field is used for records that are executed in the context of a persistent task. + + AuditEventRecordType.taskOID + @@ -149,6 +167,9 @@ Identifier of a host that generated the audit record. This is the host name corresponding to a network interface that accepted the HTTP request. + + AuditEventRecordType.hostIdentifier + @@ -156,6 +177,9 @@ TODO + + AuditEventRecordType.nodeIdentifier + @@ -163,6 +187,9 @@ TODO + + AuditEventRecordType.remoteHostAddress + @@ -177,6 +204,7 @@ c:UserType + AuditEventRecordType.initiatorRef @@ -191,6 +219,7 @@ c:UserType + AuditEventRecordType.attorneyRef @@ -201,6 +230,9 @@ In case that operation targets more than one object, the "primary" or the most important is recorded (e.g. the user object) + + AuditEventRecordType.targetRef + @@ -212,6 +244,7 @@ c:UserType + AuditEventRecordType.targetOwnerRef @@ -221,6 +254,9 @@ Type of audit event. It describes what kind of operation was executed (adding an object, modification, user login, ...) + + AuditEventRecordType.eventType + @@ -228,14 +264,26 @@ Stage of event processing: request or execution. + + AuditEventRecordType.eventStage + + + + + + + AuditEventRecordType.delta + - The channel that was the source of the operation that the record describes + + AuditEventRecordType.channel + @@ -243,6 +291,9 @@ Operation outcome (success, failure) + + AuditEventRecordType.outcome + @@ -250,6 +301,9 @@ result (e.g. number of entries, returned object, business result of workflow task or process instance - approved, rejected) + + AuditEventRecordType.result + @@ -257,6 +311,9 @@ TODO + + AuditEventRecordType.parameter + @@ -264,6 +321,9 @@ TODO + + AuditEventRecordType.message + @@ -277,6 +337,7 @@ 4.2 + AuditEventRecordType.changedItem @@ -287,6 +348,7 @@ 3.6 + AuditEventRecordType.property @@ -297,6 +359,7 @@ 3.6 + AuditEventRecordType.reference @@ -307,6 +370,7 @@ 4.0 + AuditEventRecordType.resourceOid @@ -317,7 +381,8 @@ 4.0 - true + true + AuditEventRecordType.customColumnProperty diff --git a/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd b/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd index aeff665cf1d..93a0f34909a 100755 --- a/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd +++ b/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd @@ -17549,6 +17549,7 @@ ObjectCollectionType.auditSearch true + true @@ -17965,6 +17966,7 @@ 4.0 true + true diff --git a/model/model-api/src/main/java/com/evolveum/midpoint/model/api/interaction/DashboardService.java b/model/model-api/src/main/java/com/evolveum/midpoint/model/api/interaction/DashboardService.java index 27486942d1d..3f9ac75be24 100644 --- a/model/model-api/src/main/java/com/evolveum/midpoint/model/api/interaction/DashboardService.java +++ b/model/model-api/src/main/java/com/evolveum/midpoint/model/api/interaction/DashboardService.java @@ -22,6 +22,7 @@ import com.evolveum.midpoint.util.exception.ObjectNotFoundException; import com.evolveum.midpoint.util.exception.SchemaException; import com.evolveum.midpoint.util.exception.SecurityViolationException; +import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType; import com.evolveum.midpoint.xml.ns._public.common.common_3.*; import javax.xml.namespace.QName; @@ -40,7 +41,7 @@ List> searchObjectFromCollection(CollectionRefSpecificat Collection> options, ExpressionType condition, Task task, OperationResult result) throws SchemaException, ObjectNotFoundException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException; - List searchObjectFromCollection(CollectionRefSpecificationType collectionConfig, ExpressionType condition, Task task, OperationResult result) + List searchObjectFromCollection(CollectionRefSpecificationType collectionConfig, ExpressionType condition, Task task, OperationResult result) throws SchemaException, ObjectNotFoundException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException; diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/util/DefaultColumnUtils.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/util/DefaultColumnUtils.java index ffa9d9f410a..564d0091569 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/util/DefaultColumnUtils.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/util/DefaultColumnUtils.java @@ -13,19 +13,20 @@ import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import com.evolveum.midpoint.prism.Containerable; +import com.evolveum.midpoint.prism.PrismContainer; + +import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordCustomColumnPropertyType; + import com.google.common.collect.ImmutableMap; import org.jetbrains.annotations.Nullable; -import com.evolveum.midpoint.audit.api.AuditEventRecord; import com.evolveum.midpoint.common.LocalizationService; -import com.evolveum.midpoint.prism.PrismObject; -import com.evolveum.midpoint.prism.path.ItemName; import com.evolveum.midpoint.prism.path.ItemPath; import com.evolveum.midpoint.schema.GetOperationOptions; import com.evolveum.midpoint.schema.GetOperationOptionsBuilder; import com.evolveum.midpoint.schema.SchemaHelper; import com.evolveum.midpoint.schema.SelectorOptions; -import com.evolveum.midpoint.schema.constants.AuditConstants; import com.evolveum.midpoint.task.api.TaskUtil; import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; @@ -67,23 +68,6 @@ public class DefaultColumnUtils { .put(ItemPath.create(ResourceType.F_CONNECTOR_REF, ConnectorType.F_CONNECTOR_VERSION), "ConnectorType.connectorVersion") .build()) - .put(AuditEventRecordType.class, ImmutableMap.builder() - .put(new ItemName(AuditConstants.TIME_COLUMN), AuditConstants.TIME_COLUMN_KEY) - .put(new ItemName(AuditConstants.INITIATOR_COLUMN), AuditConstants.INITIATOR_COLUMN_KEY) - .put(new ItemName(AuditConstants.EVENT_STAGE_COLUMN), AuditConstants.EVENT_STAGE_COLUMN_KEY) - .put(new ItemName(AuditConstants.EVENT_TYPE_COLUMN), AuditConstants.EVENT_TYPE_COLUMN_KEY) - .put(new ItemName(AuditConstants.TARGET_COLUMN), AuditConstants.TARGET_COLUMN_KEY) - .put(new ItemName(AuditConstants.OUTCOME_COLUMN), AuditConstants.OUTCOME_COLUMN_KEY) - .put(new ItemName(AuditConstants.MESSAGE_COLUMN), AuditConstants.MESSAGE_COLUMN_KEY) - .put(new ItemName(AuditConstants.DELTA_COLUMN), AuditConstants.DELTA_COLUMN_KEY) - .put(new ItemName(AuditConstants.TARGET_OWNER_COLUMN), AuditConstants.TARGET_OWNER_COLUMN_KEY) - .put(new ItemName(AuditConstants.CHANNEL_COLUMN), AuditConstants.CHANNEL_COLUMN_KEY) - .put(new ItemName(AuditConstants.TASK_OID_COLUMN), AuditConstants.TASK_OID_COLUMN_KEY) - .put(new ItemName(AuditConstants.NODE_IDENTIFIER_COLUMN), AuditConstants.NODE_IDENTIFIER_COLUMN_KEY) - .put(new ItemName(AuditConstants.ATTORNEY_COLUMN), AuditConstants.ATTORNEY_COLUMN_KEY) - .put(new ItemName(AuditConstants.RESULT_COLUMN), AuditConstants.RESULT_COLUMN_KEY) - .put(new ItemName(AuditConstants.RESOURCE_OID_COLUMN), AuditConstants.RESOURCE_OID_COLUMN_KEY) - .build()) .build(); NUMBER_COLUMNS = Collections.singletonList( @@ -92,14 +76,14 @@ public class DefaultColumnUtils { OBJECT_COLUMNS_DEF = Collections.singletonList(ObjectType.F_NAME); DEFAULT_AUDIT_COLUMNS_DEF = Arrays.asList( - new ItemName(AuditConstants.TIME_COLUMN), - new ItemName(AuditConstants.INITIATOR_COLUMN), - new ItemName(AuditConstants.EVENT_STAGE_COLUMN), - new ItemName(AuditConstants.EVENT_TYPE_COLUMN), - new ItemName(AuditConstants.TARGET_COLUMN), - new ItemName(AuditConstants.OUTCOME_COLUMN), - new ItemName(AuditConstants.MESSAGE_COLUMN), - new ItemName(AuditConstants.DELTA_COLUMN) + AuditEventRecordType.F_TIMESTAMP, + AuditEventRecordType.F_INITIATOR_REF, + AuditEventRecordType.F_EVENT_STAGE, + AuditEventRecordType.F_EVENT_TYPE, + AuditEventRecordType.F_TARGET_REF, + AuditEventRecordType.F_OUTCOME, + AuditEventRecordType.F_MESSAGE, + AuditEventRecordType.F_DELTA ); COLUMNS_DEF = ImmutableMap., List>builder() @@ -280,52 +264,9 @@ private static void createColumns(GuiObjectListViewType v } } - public static Object getObjectByAuditColumn(AuditEventRecord record, ItemPath path) { - switch (path.toString()) { - case AuditConstants.TIME_COLUMN: - return record.getTimestamp(); - case AuditConstants.INITIATOR_COLUMN: - return record.getInitiatorRef(); - case AuditConstants.EVENT_STAGE_COLUMN: - return record.getEventStage(); - case AuditConstants.EVENT_TYPE_COLUMN: - return record.getEventType(); - case AuditConstants.TARGET_COLUMN: - return record.getTargetRef(); - case AuditConstants.TARGET_OWNER_COLUMN: - return record.getTargetOwnerRef(); - case AuditConstants.CHANNEL_COLUMN: - return record.getChannel(); - case AuditConstants.OUTCOME_COLUMN: - return record.getOutcome(); - case AuditConstants.MESSAGE_COLUMN: - return record.getMessage(); - case AuditConstants.DELTA_COLUMN: - return record.getDeltas(); - case AuditConstants.TASK_OID_COLUMN: - return record.getTaskOid(); - case AuditConstants.NODE_IDENTIFIER_COLUMN: - return record.getNodeIdentifier(); - case AuditConstants.ATTORNEY_COLUMN: - return record.getAttorneyRef(); - case AuditConstants.RESULT_COLUMN: - return record.getResult(); - case AuditConstants.RESOURCE_OID_COLUMN: - return record.getResourceOids(); - default: - String value = record.getCustomColumnProperty().get(path.toString()); - if (value != null) { - return value; - } - - LOGGER.error("Unknown name of column for AuditReport " + path); - throw new IllegalArgumentException("Unknown name of column for AuditReport " + path); - } - } - - public static String processSpecialColumn( - ItemPath itemPath, PrismObject object, LocalizationService localization) { - @Nullable Class type = object.getCompileTimeClass(); + public static String processSpecialColumn( + ItemPath itemPath, PrismContainer object, LocalizationService localization) { + @Nullable Class type = object.getCompileTimeClass(); if (type == null || itemPath == null) { return null; } @@ -362,12 +303,18 @@ public static String processSpecialColumn( Object[] params = localizationObject.isEmpty() ? null : localizationObject.toArray(); return localization.translate(key, params, Locale.getDefault(), key); } + } else if (type.isAssignableFrom(AuditEventRecordType.class)) { + for (AuditEventRecordCustomColumnPropertyType customColumn : ((AuditEventRecordType)object.getValue().asContainerable()).getCustomColumnProperty()) { + if (customColumn.getName().equals(itemPath.toString())) { + return customColumn.getValue(); + } + } } return null; } - public static boolean isSpecialColumn(ItemPath itemPath, PrismObject object) { - @Nullable Class type = object.getCompileTimeClass(); + public static boolean isSpecialColumn(ItemPath itemPath, PrismContainer object) { + @Nullable Class type = object.getCompileTimeClass(); if (type == null || itemPath == null) { return false; } @@ -379,6 +326,12 @@ public static boolean isSpecialColumn(ItemPath itemPath, || itemPath.equivalent(TaskType.F_PROGRESS)) { return true; } + } else if (type.isAssignableFrom(AuditEventRecordType.class)) { + for (AuditEventRecordCustomColumnPropertyType customColumn : ((AuditEventRecordType)object.getValue().asContainerable()).getCustomColumnProperty()) { + if (customColumn.getName().equals(itemPath.toString())) { + return true; + } + } } return false; } diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/DashboardServiceImpl.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/DashboardServiceImpl.java index 9f6791ccf4a..db3d9c5aae9 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/DashboardServiceImpl.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/DashboardServiceImpl.java @@ -15,14 +15,20 @@ import com.evolveum.midpoint.prism.PrismPropertyValue; import com.evolveum.midpoint.prism.query.ObjectFilter; import com.evolveum.midpoint.schema.GetOperationOptions; +import com.evolveum.midpoint.schema.GetOperationOptionsBuilder; +import com.evolveum.midpoint.schema.SchemaHelper; import com.evolveum.midpoint.schema.SelectorOptions; import com.evolveum.midpoint.schema.util.MiscSchemaUtil; +import com.evolveum.midpoint.schema.util.ObjectQueryUtil; import com.evolveum.midpoint.util.QNameUtil; +import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType; + import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -75,9 +81,11 @@ public class DashboardServiceImpl implements DashboardService { @Autowired private ExpressionFactory expressionFactory; @Autowired private ModelObjectResolver objectResolver; @Autowired private CollectionProcessor collectionProcessor; + @Autowired private SchemaHelper schemaHelper; @Override - public DashboardWidget createWidgetData(DashboardWidgetType widget, Task task, OperationResult result) throws SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException, ObjectNotFoundException { + public DashboardWidget createWidgetData(DashboardWidgetType widget, Task task, OperationResult result) + throws SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException, ObjectNotFoundException { Validate.notNull(widget, "Widget is null"); @@ -171,7 +179,8 @@ public DashboardWidgetSourceTypeType getSourceType(DashboardWidgetType widget) { return widget.getData().getSourceType(); } - private String getNumberMessage(DashboardWidgetType widget, DashboardWidget data, Task task, OperationResult result) throws SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException, ObjectNotFoundException { + private String getNumberMessage(DashboardWidgetType widget, DashboardWidget data, Task task, OperationResult result) + throws SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException, ObjectNotFoundException { DashboardWidgetSourceTypeType sourceType = getSourceType(widget); DashboardWidgetPresentationType presentation = widget.getPresentation(); switch (sourceType) { @@ -194,7 +203,8 @@ private String getNumberMessage(DashboardWidgetType widget, DashboardWidget data return null; } - private String generateNumberMessageForObject(DashboardWidgetType widget, DashboardWidget data, Task task, OperationResult result) throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { + private String generateNumberMessageForObject(DashboardWidgetType widget, DashboardWidget data, Task task, OperationResult result) + throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { ObjectType object = getObjectFromObjectRef(widget, task, result); if(object == null) { return null; @@ -202,43 +212,141 @@ private String generateNumberMessageForObject(DashboardWidgetType widget, Dashbo return generateNumberMessage(widget, createVariables(object.asPrismObject(), null, null), data); } - private String generateNumberMessageForAuditSearch(DashboardWidgetType widget, DashboardWidget data, Task task, OperationResult result) throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { + private String generateNumberMessageForAuditSearch(DashboardWidgetType widget, DashboardWidget data, Task task, OperationResult result) + throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { ObjectCollectionType collection = getObjectCollectionType(widget, task, result); + CollectionRefSpecificationType collectionRef = getCollectionRefSpecificationType(widget, task, result); if(collection == null) { return null; } AuditSearchType auditSearch = collection.getAuditSearch(); - if(auditSearch == null) { - LOGGER.error("AuditSearch of ObjectCollection is not found in widget " + + SearchFilterType filter = collection.getFilter(); + Integer value = 0; + Integer domainValue = null; + if (filter != null) { + value = countAuditEvents(collectionRef, collection, task, result); + if (value == null) { + return null; + } + if (collection.getDomain() != null && collection.getDomain().getCollectionRef() != null + && collection.getDomain().getCollectionRef().getOid() != null) { + @NotNull PrismObject domainCollection = modelService.getObject(ObjectCollectionType.class, collection.getDomain().getCollectionRef().getOid(), + null, task, result); + domainValue = countAuditEvents(collection.getDomain(), domainCollection.asObjectable(), task, result); + } + } else if (auditSearch != null) { + if (auditSearch.getRecordQuery() == null) { + LOGGER.error("RecordQuery of auditSearch is not defined in widget " + + widget.getIdentifier()); + return null; + } + + Map parameters = new HashMap<>(); + String query = getQueryForCount(createQuery(collection, + parameters, false, clock)); + LOGGER.debug("Parameters for select: " + parameters); + value = (int) auditService.countObjects( + query, parameters); + domainValue = null; + if (auditSearch.getDomainQuery() == null) { + LOGGER.error("DomainQuery of auditSearch is not defined"); + } else { + parameters = new HashMap<>(); + query = getQueryForCount(createQuery(collection, + parameters, true, clock)); + LOGGER.debug("Parameters for select: " + parameters); + domainValue = (int) auditService.countObjects( + query, parameters); + } + } else { + LOGGER.error("Filter or auditSearch of ObjectCollection is not found in widget " + widget.getIdentifier()); return null; } - if(auditSearch.getRecordQuery() == null) { - LOGGER.error("RecordQuery of auditSearch is not defined in widget " + - widget.getIdentifier()); + LOGGER.debug("Value: {}, Domain value: {}", value, domainValue); + IntegerStatType statType = generateIntegerStat(value, domainValue); + return generateNumberMessage(widget, createVariables(null, statType, null), data); + } + + private Integer countAuditEvents(CollectionRefSpecificationType collectionRef, ObjectCollectionType collection, Task task, OperationResult result) + throws SchemaException, ConfigurationException, ObjectNotFoundException, CommunicationException, SecurityViolationException, ExpressionEvaluationException { + + if (collectionRef == null || collectionRef.getCollectionRef() == null || collectionRef.getCollectionRef().getOid() == null) { return null; } - Map parameters = new HashMap<>(); - String query = getQueryForCount(createQuery(collection, - parameters, false, clock)); - LOGGER.debug("Parameters for select: " + parameters); - int value = (int) auditService.countObjects( - query, parameters); - Integer domainValue = null; - if(auditSearch.getDomainQuery() == null) { - LOGGER.error("DomainQuery of auditSearch is not defined"); + if (collection.getType() != null && !QNameUtil.match(collection.getType(), AuditEventRecordType.COMPLEX_TYPE)) { + LOGGER.error("Unsupported type for audit query " + collection.getType() + " in object collection " + collection.getName()); + return null; + } + + SearchFilterType filter = null; + if (collection != null) { + filter = collection.getFilter(); + } + ObjectFilter objectFilter = combineAuditFilter(collectionRef, filter, task, result); + ObjectQuery query; + if (objectFilter == null) { + query = prismContext.queryFor(AuditEventRecordType.class).build(); } else { - parameters = new HashMap<>(); - query = getQueryForCount(createQuery(collection, - parameters, true, clock)); - LOGGER.debug("Parameters for select: " + parameters); - domainValue = (int) auditService.countObjects( - query, parameters); + query = prismContext.queryFactory().createQuery(); + ObjectFilter evaluatedFilter = ExpressionUtil.evaluateFilterExpressions(objectFilter, new ExpressionVariables(), MiscSchemaUtil.getExpressionProfile(), + expressionFactory, prismContext, "collection filter", task, result); + query.setFilter(evaluatedFilter); } - LOGGER.debug("Value: {}, Domain value: {}", value, domainValue); - IntegerStatType statType = generateIntegerStat(value, domainValue); - return generateNumberMessage(widget, createVariables(null, statType, null), data); + @NotNull Collection> option = combineAuditOption(collectionRef, collection, task, result); + + return auditService.countObjects(query, option, result); + } + + private @NotNull Collection> combineAuditOption(CollectionRefSpecificationType collectionRef, ObjectCollectionType collection, Task task, OperationResult result) + throws CommunicationException, ObjectNotFoundException, SchemaException, SecurityViolationException, ConfigurationException, ExpressionEvaluationException { + + List> collectionOptions; + if (collection != null) { + collectionOptions = MiscSchemaUtil.optionsTypeToOptions(collection.getGetOptions(), prismContext); + } else { + @NotNull PrismObject collectionFromRef = modelService.getObject(ObjectCollectionType.class, collectionRef.getCollectionRef().getOid(), null, task, result); + collectionOptions = MiscSchemaUtil.optionsTypeToOptions(collectionFromRef.asObjectable().getGetOptions(), prismContext); + } + GetOperationOptionsBuilder optionsBuilder = schemaHelper.getOperationOptionsBuilder().setFrom(collectionOptions); + if (collectionRef.getBaseCollectionRef() != null && collectionRef.getBaseCollectionRef().getCollectionRef() != null + && collectionRef.getBaseCollectionRef().getCollectionRef().getOid() != null) { + @NotNull PrismObject baseCollection = modelService.getObject(ObjectCollectionType.class, collectionRef.getCollectionRef().getOid(), null, task, result); + List> baseCollectionOptions = MiscSchemaUtil.optionsTypeToOptions(baseCollection.asObjectable().getGetOptions(), prismContext); + optionsBuilder.mergeFrom(baseCollectionOptions); + } + return optionsBuilder.build(); + + } + + private ObjectFilter combineAuditFilter(CollectionRefSpecificationType collectionRef, SearchFilterType baseFilter, Task task, OperationResult result) + throws CommunicationException, ObjectNotFoundException, SchemaException, SecurityViolationException, ConfigurationException, ExpressionEvaluationException { + SearchFilterType filter = baseFilter; + if (filter == null) { + @NotNull PrismObject collection = modelService.getObject(ObjectCollectionType.class, collectionRef.getCollectionRef().getOid(), null, task, result); + filter = collection.asObjectable().getFilter(); + } + if (collectionRef.getBaseCollectionRef() != null && collectionRef.getBaseCollectionRef().getCollectionRef() != null + && collectionRef.getBaseCollectionRef().getCollectionRef().getOid() != null) { + @NotNull PrismObject baseCollection = modelService.getObject(ObjectCollectionType.class, collectionRef.getCollectionRef().getOid(), null, task, result); + if (filter == null && baseCollection.asObjectable().getFilter() == null) { + return null; + } else if (filter == null) { + return prismContext.getQueryConverter().parseFilter(baseCollection.asObjectable().getFilter(), AuditEventRecordType.class); + } else if (baseCollection.asObjectable().getFilter() == null) { + return prismContext.getQueryConverter().parseFilter(filter, AuditEventRecordType.class); + } else { + ObjectFilter baseFilterFromCollection = prismContext.getQueryConverter().parseFilter(baseCollection.asObjectable().getFilter(), AuditEventRecordType.class); + ObjectFilter baseObjectFilter = prismContext.getQueryConverter().parseFilter(filter, AuditEventRecordType.class); + ObjectQueryUtil.filterAnd(baseFilterFromCollection, baseObjectFilter, prismContext); + return prismContext.getQueryConverter().parseFilter(filter, AuditEventRecordType.class); + } + } + if (filter == null) { + return null; + } + return prismContext.getQueryConverter().parseFilter(filter, AuditEventRecordType.class); } private String getQueryForCount(String query) { @@ -436,24 +544,41 @@ public List> searchObjectFromCollection(CollectionRefSpe } @Override - public List searchObjectFromCollection(CollectionRefSpecificationType collectionConfig, ExpressionType condition, Task task, OperationResult result) + public List searchObjectFromCollection(CollectionRefSpecificationType collectionConfig, ExpressionType condition, Task task, OperationResult result) throws SchemaException, ObjectNotFoundException, SecurityViolationException, CommunicationException, ConfigurationException, ExpressionEvaluationException { - List auditRecords = new ArrayList<>(); + List auditRecords = new ArrayList<>(); if (collectionConfig.getCollectionRef() != null) { ObjectReferenceType ref = collectionConfig.getCollectionRef(); Class refType = prismContext.getSchemaRegistry().determineClassForType(ref.getType()); ObjectCollectionType collection = (ObjectCollectionType) modelService .getObject(refType, ref.getOid(), null, task, result).asObjectable(); - if (collection.getAuditSearch() != null) { - Map parameters = new HashMap<>(); - String query = DashboardUtils.getQueryForListRecords(DashboardUtils.createQuery(collection, parameters, false, clock)); - auditRecords = auditService.listRecords(query, parameters, result); + if (collection != null && collection.getFilter() != null) { + ObjectFilter objectFilter = combineAuditFilter(collectionConfig, collection.getFilter(), task, result); + ObjectFilter evaluatedFilter = ExpressionUtil.evaluateFilterExpressions(objectFilter, new ExpressionVariables(), MiscSchemaUtil.getExpressionProfile(), + expressionFactory, prismContext, "collection filter", task, result); + ObjectQuery query = prismContext.queryFactory().createQuery(); + query.setFilter(evaluatedFilter); + @NotNull Collection> option = combineAuditOption(collectionConfig, collection, task, result); + auditRecords.addAll(auditService.searchObjects(query, option, result)); if (auditRecords == null) { auditRecords = new ArrayList<>(); } if (condition != null) { return evaluateCondition(condition, auditRecords, task, result); } + } else if (collection != null && collection.getAuditSearch() != null) { + Map parameters = new HashMap<>(); + String query = DashboardUtils.getQueryForListRecords(DashboardUtils.createQuery(collection, parameters, false, clock)); + List oldAuditRecords = auditService.listRecords(query, parameters, result); + if (oldAuditRecords == null) { + oldAuditRecords = new ArrayList<>(); + } + for (AuditEventRecord auditRecord : oldAuditRecords) { + auditRecords.add(auditRecord.createAuditEventRecordType(true)); + } + if (condition != null) { + return evaluateCondition(condition, auditRecords, task, result); + } } } return auditRecords; @@ -475,7 +600,8 @@ private List evaluateCondition(ExpressionType condition, L } @Override - public ObjectCollectionType getObjectCollectionType(DashboardWidgetType widget, Task task, OperationResult result) throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { + public ObjectCollectionType getObjectCollectionType(DashboardWidgetType widget, Task task, OperationResult result) + throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { if (isCollectionRefOfCollectionNull(widget)) { return null; } @@ -491,7 +617,8 @@ public CollectionRefSpecificationType getCollectionRefSpecificationType(Dashboar return widget.getData().getCollection(); } - private ObjectType getObjectFromObjectRef(DashboardWidgetType widget, Task task, OperationResult result) throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { + private ObjectType getObjectFromObjectRef(DashboardWidgetType widget, Task task, OperationResult result) + throws ObjectNotFoundException, SchemaException, CommunicationException, ConfigurationException, SecurityViolationException, ExpressionEvaluationException { if(isDataNull(widget)) { return null; } diff --git a/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/CsvController.java b/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/CsvController.java index 98dbd2c06e9..c09fe92822b 100644 --- a/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/CsvController.java +++ b/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/CsvController.java @@ -23,9 +23,11 @@ import com.evolveum.midpoint.schema.util.MiscSchemaUtil; import com.evolveum.midpoint.task.api.RunningTask; import com.evolveum.midpoint.task.api.Task; +import com.evolveum.midpoint.util.QNameUtil; import com.evolveum.midpoint.util.exception.*; import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType; import com.evolveum.midpoint.xml.ns._public.common.common_3.*; import com.evolveum.prism.xml.ns._public.types_3.ItemPathType; @@ -131,8 +133,7 @@ public byte[] processCollection(String nameOfReport, ObjectCollectionReportEngin CompiledObjectCollectionView compiledCollection = createCompiledView(collectionConfig, collection); byte[] csvFile; - boolean isAuditCollection = collection != null && collection.getAuditSearch() != null; - if (!isAuditCollection) { + if (!isAuditCollection(collection)) { csvFile = createTableBoxForObjectView(collectionRefSpecification, compiledCollection, collectionConfig.getCondition(), task, result); } else { @@ -192,7 +193,7 @@ private CompiledObjectCollectionView createCompiledView (ObjectCollectionReportE private byte[] createTableForAuditView(CollectionRefSpecificationType collectionRef, CompiledObjectCollectionView compiledCollection, ExpressionType condition, Task task, OperationResult result) throws CommunicationException, ObjectNotFoundException, SchemaException, SecurityViolationException, ConfigurationException, ExpressionEvaluationException { - List auditRecords = getReportService().getDashboardService().searchObjectFromCollection(collectionRef, condition, task, result); + List auditRecords = getReportService().getDashboardService().searchObjectFromCollection(collectionRef, condition, task, result); if (compiledCollection.getColumns().isEmpty()) { getReportService().getModelInteractionService().applyView(compiledCollection, DefaultColumnUtils.getDefaultAuditEventsView()); @@ -201,17 +202,11 @@ private byte[] createTableForAuditView(CollectionRefSpecificationType collection List> records = new ArrayList<>(); List columns = MiscSchemaUtil.orderCustomColumns(compiledCollection.getColumns()); - + PrismContainerDefinition def = getReportService().getPrismContext().getSchemaRegistry() + .findItemDefinitionByCompileTimeClass(AuditEventRecordType.class, PrismContainerDefinition.class); columns.forEach(column -> { Validate.notNull(column.getName(), "Name of column is null"); - - DisplayType columnDisplay = column.getDisplay(); - String label; - if(columnDisplay != null && columnDisplay.getLabel() != null) { - label = getMessage(columnDisplay.getLabel().getOrig()); - } else { - label = column.getName(); - } + String label = getColumnLabel(column, def); headers.add(label); }); @@ -220,12 +215,17 @@ private byte[] createTableForAuditView(CollectionRefSpecificationType collection task.setExpectedTotal((long) auditRecords.size()); recordProgress(task, i, result, LOGGER); - for (AuditEventRecord auditRecord : auditRecords) { + for (AuditEventRecordType auditRecord : auditRecords) { List items = new ArrayList<>(); columns.forEach(column -> { ExpressionType expression = column.getExport() != null ? column.getExport().getExpression() : null; ItemPath path = column.getPath() == null ? null : column.getPath().getItemPath(); - items.add(getStringValueByAuditColumn(auditRecord, path, expression, task, result)); + try { + items.add(getRealValueAsString(column, getAuditRecordAsContainer(auditRecord), path, expression, task, result) + ); + } catch (SchemaException e) { + LOGGER.error("Couldn't create singleValueContainer for audit record " + auditRecord); + } }); records.add(items); i++; @@ -326,19 +326,12 @@ private byte[] createTableBoxForObjectView(CollectionRefSpecificationType collec } @Override - protected String getRealValueAsString(GuiObjectColumnType column, PrismObject object, ItemPath itemPath, ExpressionType expression, Task task, OperationResult result) { + protected String getRealValueAsString(GuiObjectColumnType column, PrismContainer object, ItemPath itemPath, ExpressionType expression, Task task, OperationResult result) { String value = super.getRealValueAsString(column, object, itemPath, expression, task, result); value = removeNewLine(value); return value; } - protected String getStringValueByAuditColumn(AuditEventRecord record, ItemPath path, - ExpressionType expression, Task task, OperationResult result) { - String value = super.getStringValueByAuditColumn(record, path, expression, task, result); - value = removeNewLine(value); - return value; - } - @Override public void importCollectionReport(ReportType report, VariablesMap variables, RunningTask task, OperationResult result) { if (report.getObjectCollection() != null) { diff --git a/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/FileFormatController.java b/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/FileFormatController.java index fd5d8f8dd74..a04d625072e 100644 --- a/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/FileFormatController.java +++ b/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/FileFormatController.java @@ -8,12 +8,14 @@ import java.io.IOException; import java.util.*; +import javax.xml.datatype.XMLGregorianCalendar; import javax.xml.namespace.QName; +import com.evolveum.midpoint.schema.DeltaConvertor; + import com.google.common.collect.ImmutableSet; import org.jetbrains.annotations.NotNull; -import com.evolveum.midpoint.audit.api.AuditEventRecord; import com.evolveum.midpoint.model.api.authentication.CompiledObjectCollectionView; import com.evolveum.midpoint.model.common.util.DefaultColumnUtils; import com.evolveum.midpoint.prism.*; @@ -24,7 +26,6 @@ import com.evolveum.midpoint.report.impl.ReportServiceImpl; import com.evolveum.midpoint.report.impl.ReportUtils; import com.evolveum.midpoint.schema.ObjectDeltaOperation; -import com.evolveum.midpoint.schema.constants.AuditConstants; import com.evolveum.midpoint.schema.constants.ExpressionConstants; import com.evolveum.midpoint.schema.expression.ExpressionProfile; import com.evolveum.midpoint.schema.expression.VariablesMap; @@ -106,7 +107,7 @@ protected String getMessage(String key, Object... params) { return getReportService().getLocalizationService().translate(key, params, Locale.getDefault(), key); } - protected String getRealValueAsString(GuiObjectColumnType column, PrismObject object, ItemPath itemPath, + protected String getRealValueAsString(GuiObjectColumnType column, PrismContainer object, ItemPath itemPath, ExpressionType expression, Task task, OperationResult result) { Item valueObject = object; @@ -173,22 +174,35 @@ protected String getRealValueAsString(GuiObjectColumnType column, PrismObject values) { + private String processListOfRealValues(Collection values) { StringBuilder sb = new StringBuilder(); values.forEach(value -> { if (!sb.toString().isEmpty()) { appendMultivalueDelimiter(sb); } if (value instanceof PrismPropertyValue) { - Object realObject = ((PrismPropertyValue) value).getRealValue(); + String stringValue; + O realObject = ((PrismPropertyValue) value).getRealValue(); if (realObject == null) { - realObject = ""; + stringValue = ""; } else if (realObject instanceof Collection) { - realObject = processListOfRealValues((Collection) realObject); + stringValue = processListOfRealValues((Collection) realObject); + } else if (realObject instanceof Enum){ + stringValue = ReportUtils.prettyPrintForReport((Enum) realObject); + } else if (realObject instanceof XMLGregorianCalendar) { + stringValue = ReportUtils.prettyPrintForReport((XMLGregorianCalendar) realObject); + } else if (realObject instanceof ObjectDeltaOperationType) { + try { + ObjectDeltaOperation convertedDelta = DeltaConvertor.createObjectDeltaOperation((ObjectDeltaOperationType) realObject, getReportService().getPrismContext()); + stringValue = ReportUtils.printDelta(convertedDelta); + } catch (SchemaException e) { + LOGGER.error("Couldn't convert delta from ObjectDeltaOperationType to ObjectDeltaOperation " + realObject.toString()); + stringValue = ""; + } } else { - realObject = ReportUtils.prettyPrintForReport(realObject); + stringValue = ReportUtils.prettyPrintForReport(realObject); } - sb.append(realObject); + sb.append(stringValue); } else if (value instanceof PrismReferenceValue) { sb.append(getObjectNameFromRef(((PrismReferenceValue) value).getRealValue())); } else { @@ -323,98 +337,6 @@ protected PrismObject getObjectFromReference(Referencable ref) { return object; } - protected String getStringValueByAuditColumn(AuditEventRecord record, ItemPath path, - ExpressionType expression, Task task, OperationResult result) { - if (expression == null) { - return getStringValueByAuditColumn(record, path); - } - Object object; - if (path == null) { - object = record; - } else { - object = DefaultColumnUtils.getObjectByAuditColumn(record, path); - } - Object value = evaluateExportExpression(expression, object, task, result); - if (value instanceof Collection) { - return processListOfRealValues((Collection) value); - } - return processListOfRealValues(Collections.singletonList(value)); - } - - private String getStringValueByAuditColumn(AuditEventRecord record, ItemPath path) { - switch (path.toString()) { - case AuditConstants.TIME_COLUMN: - return ReportUtils.prettyPrintForReport(new Date(record.getTimestamp())); - case AuditConstants.INITIATOR_COLUMN: - return record.getInitiatorRef() == null ? "" - : getObjectNameFromRef(record.getInitiatorRef().getRealValue()); - case AuditConstants.EVENT_STAGE_COLUMN: - return record.getEventStage() == null ? "" : ReportUtils.prettyPrintForReport(record.getEventStage()); - case AuditConstants.EVENT_TYPE_COLUMN: - return record.getEventType() == null ? "" : ReportUtils.prettyPrintForReport(record.getEventType()); - case AuditConstants.TARGET_COLUMN: - return record.getTargetRef() == null ? "" - : getObjectNameFromRef(record.getTargetRef().getRealValue()); - case AuditConstants.TARGET_OWNER_COLUMN: - return record.getTargetOwnerRef() == null ? "" - : getObjectNameFromRef(record.getTargetOwnerRef().getRealValue()); - case AuditConstants.CHANNEL_COLUMN: - return record.getChannel() == null ? "" : ReportUtils.prettyPrintForReport(QNameUtil.uriToQName(record.getChannel())); - case AuditConstants.OUTCOME_COLUMN: - return record.getOutcome() == null ? "" : ReportUtils.prettyPrintForReport(record.getOutcome()); - case AuditConstants.MESSAGE_COLUMN: - return record.getMessage() == null ? "" : record.getMessage(); - case AuditConstants.DELTA_COLUMN: - if (record.getDeltas().isEmpty()) { - return ""; - } - StringBuilder sbDelta = new StringBuilder(); - Collection> deltas = record.getDeltas(); - Iterator> iterator = deltas.iterator(); - int index = 0; - while (iterator.hasNext()) { - ObjectDeltaOperation delta = iterator.next(); - sbDelta.append(ReportUtils.printDelta(delta)); - if ((index + 1) != deltas.size()) { - sbDelta.append("\n"); - } - index++; - } - return sbDelta.toString(); - case AuditConstants.TASK_OID_COLUMN: - return record.getTaskOid() == null ? "" : record.getTaskOid(); - case AuditConstants.NODE_IDENTIFIER_COLUMN: - return record.getNodeIdentifier() == null ? "" : record.getNodeIdentifier(); - case AuditConstants.ATTORNEY_COLUMN: - return record.getAttorneyRef() == null ? "" - : getObjectNameFromRef(record.getAttorneyRef().getRealValue()); - case AuditConstants.RESULT_COLUMN: - return record.getResult() == null ? "" : record.getResult(); - case AuditConstants.RESOURCE_OID_COLUMN: - Set resourceOids = record.getResourceOids(); - if (resourceOids == null || resourceOids.isEmpty()) { - return ""; - } - StringBuilder sb = new StringBuilder(); - int i = 1; - for (String oid : resourceOids) { - sb.append(oid); - if (i != resourceOids.size()) { - sb.append("\n"); - } - i++; - } - return sb.toString(); - default: - if (record.getCustomColumnProperty().containsKey(path.toString())) { - return record.getCustomColumnProperty().get(path.toString()); - } else { - LOGGER.error("Unknown name of column for AuditReport " + path); - throw new IllegalArgumentException("Unknown name of column for AuditReport " + path); - } - } - } - protected QName resolveTypeQname(CollectionRefSpecificationType collectionRef, CompiledObjectCollectionView compiledCollection) { QName type; if (collectionRef.getCollectionRef() != null) { @@ -443,6 +365,17 @@ protected Class resolveType(CollectionRefSpecificationType collectio .getCompileTimeClassForObjectType(type); } + protected boolean isAuditCollection(ObjectCollectionType collection) { + return collection != null + && (collection.getAuditSearch() != null || QNameUtil.match(AuditEventRecordType.COMPLEX_TYPE, collection.getType())); + } + + protected PrismContainer getAuditRecordAsContainer(AuditEventRecordType record) throws SchemaException { + PrismContainerValue prismValue = record.asPrismContainerValue(); + prismValue.setPrismContext(getReportService().getPrismContext()); + return prismValue.asSingleValuedContainer(AuditEventRecordType.COMPLEX_TYPE); + } + public abstract void importCollectionReport(ReportType report, VariablesMap listOfVariables, RunningTask task, OperationResult result); public abstract List createVariablesFromFile(ReportType report, ReportDataType reportData, boolean useImportScript, Task task, OperationResult result) throws IOException; diff --git a/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/HtmlController.java b/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/HtmlController.java index e0844e99594..a0d3807b780 100644 --- a/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/HtmlController.java +++ b/model/report-impl/src/main/java/com/evolveum/midpoint/report/impl/controller/fileformat/HtmlController.java @@ -13,10 +13,13 @@ import java.util.*; import javax.xml.namespace.QName; +import com.evolveum.midpoint.prism.PrismContainerDefinition; import com.evolveum.midpoint.prism.PrismContainerValue; import com.evolveum.midpoint.schema.expression.VariablesMap; import com.evolveum.midpoint.task.api.RunningTask; +import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType; + import j2html.TagCreator; import j2html.tags.ContainerTag; import org.apache.commons.io.IOUtils; @@ -220,8 +223,7 @@ public byte[] processCollection(String nameOfReport, ObjectCollectionReportEngin } ContainerTag tableBox; - boolean isAuditCollection = collection != null && collection.getAuditSearch() != null; - if (!isAuditCollection) { + if (!isAuditCollection(collection)) { tableBox = createTableBoxForObjectView(label, collectionRefSpecification, compiledCollection, collectionConfig.getCondition(), task, result, true); } else { @@ -293,23 +295,20 @@ private ContainerTag createTable(CompiledObjectCollectionView compiledCollection return table; } - private ContainerTag createTable(CompiledObjectCollectionView compiledCollection, List records, boolean recordProgress, + private ContainerTag createTable(CompiledObjectCollectionView compiledCollection, List records, boolean recordProgress, Task task, OperationResult result) { ContainerTag table = createTable(); ContainerTag tHead = TagCreator.thead(); ContainerTag tBody = TagCreator.tbody(); List columns = MiscSchemaUtil.orderCustomColumns(compiledCollection.getColumns()); ContainerTag trForHead = TagCreator.tr().withStyle("width: 100%;"); + PrismContainerDefinition def = getReportService().getPrismContext().getSchemaRegistry() + .findItemDefinitionByCompileTimeClass(AuditEventRecordType.class, PrismContainerDefinition.class); columns.forEach(column -> { Validate.notNull(column.getName(), "Name of column is null"); DisplayType columnDisplay = column.getDisplay(); - String label; - if (columnDisplay != null && columnDisplay.getLabel() != null) { - label = getMessage(columnDisplay.getLabel().getOrig()); - } else { - label = column.getName(); - } + String label = getColumnLabel(column, def); ContainerTag th = TagCreator.th(TagCreator.div(TagCreator.span(label).withClass("sortableLabel"))); if (columnDisplay != null) { if (StringUtils.isNotBlank(columnDisplay.getCssClass())) { @@ -329,14 +328,19 @@ private ContainerTag createTable(CompiledObjectCollectionView compiledCollection task.setExpectedTotal((long) records.size()); recordProgress(task, i, result, LOGGER); } - for (AuditEventRecord record : records) { + for (AuditEventRecordType record : records) { ContainerTag tr = TagCreator.tr(); columns.forEach(column -> { ExpressionType expression = column.getExport() != null ? column.getExport().getExpression() : null; ItemPath path = column.getPath() == null ? null : column.getPath().getItemPath(); - tr.with(TagCreator - .th(TagCreator.div(getStringValueByAuditColumn(record, path, expression, task, result)) - .withStyle("white-space: pre-wrap"))); + try { + tr.with(TagCreator + .th(TagCreator.div(getRealValueAsString(column, getAuditRecordAsContainer(record), + path, expression, task, result)) + .withStyle("white-space: pre-wrap"))); + } catch (SchemaException e) { + LOGGER.error("Couldn't create singleValueContainer for audit record " + record); + } }); tBody.with(tr); if (recordProgress) { @@ -398,7 +402,7 @@ private ContainerTag createTableBoxForAuditView( ExpressionType condition, Task task, OperationResult result, boolean recordProgress) throws CommunicationException, ObjectNotFoundException, SchemaException, SecurityViolationException, ConfigurationException, ExpressionEvaluationException { long startMillis = getReportService().getClock().currentTimeMillis(); - List records = getReportService().getDashboardService().searchObjectFromCollection(collection, condition, task, result); + List records = getReportService().getDashboardService().searchObjectFromCollection(collection, condition, task, result); if (compiledCollection.getColumns().isEmpty()) { getReportService().getModelInteractionService().applyView(compiledCollection, DefaultColumnUtils.getDefaultAuditEventsView()); diff --git a/model/report-impl/src/test/resources/common/object-collection-all-audit-records.xml b/model/report-impl/src/test/resources/common/object-collection-all-audit-records.xml index 7b88f52b125..1ae4c90b4b9 100644 --- a/model/report-impl/src/test/resources/common/object-collection-all-audit-records.xml +++ b/model/report-impl/src/test/resources/common/object-collection-all-audit-records.xml @@ -7,9 +7,18 @@ --> Audit records - - select * from m_audit_event as aer where aer.eventStage=1 - P1D - + AuditEventRecordType + + + + eventStage + execution + + + timestamp + 2020-01-01T00:00:00.000+02:00 + + + diff --git a/repo/audit-api/src/main/java/com/evolveum/midpoint/audit/api/AuditEventRecord.java b/repo/audit-api/src/main/java/com/evolveum/midpoint/audit/api/AuditEventRecord.java index 1ce93b0b8ff..25080f8e8c8 100644 --- a/repo/audit-api/src/main/java/com/evolveum/midpoint/audit/api/AuditEventRecord.java +++ b/repo/audit-api/src/main/java/com/evolveum/midpoint/audit/api/AuditEventRecord.java @@ -10,6 +10,8 @@ import java.text.SimpleDateFormat; import java.util.*; +import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordCustomColumnPropertyType; + import org.apache.commons.lang3.Validate; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -591,6 +593,14 @@ public AuditEventRecordType createAuditEventRecordType(boolean tolerateInconsist referenceEntry.getValue().forEach(v -> referenceType.getValue().add(v.toXml())); auditRecordType.getReference().add(referenceType); } + + for (Map.Entry customColumnEntry : customColumnProperty.entrySet()) { + AuditEventRecordCustomColumnPropertyType customColumn = new AuditEventRecordCustomColumnPropertyType(); + customColumn.setName(customColumnEntry.getKey()); + customColumn.setValue(customColumnEntry.getValue()); + auditRecordType.getCustomColumnProperty().add(customColumn); + } + // TODO MID-5531 convert custom properties too? What about other than string types? return auditRecordType; } From 5606a17592bb74f9424c54a28ba533601677a64a Mon Sep 17 00:00:00 2001 From: kate Date: Thu, 3 Sep 2020 15:41:19 +0200 Subject: [PATCH 02/12] save search to page storage --- .../api/component/ContainerableListPanel.java | 7 +++++++ .../gui/api/component/ObjectListPanel.java | 18 +++++++++--------- .../web/component/search/SearchFormPanel.java | 9 +++++++++ .../web/component/search/SearchPanel.java | 5 +++++ 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/component/ContainerableListPanel.java b/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/component/ContainerableListPanel.java index 396ae7be6a0..f258d09bbb1 100644 --- a/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/component/ContainerableListPanel.java +++ b/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/component/ContainerableListPanel.java @@ -95,6 +95,13 @@ protected WebMarkupContainer createHeader(String headerId) { protected void searchPerformed(ObjectQuery query, AjaxRequestTarget target) { ContainerableListPanel.this.searchPerformed(target); } + + @Override + protected void saveSearch(Search search, AjaxRequestTarget target) { + if (getPageStorage() != null) { + getPageStorage().setSearch(search); + } + } }; searchPanel.add(new VisibleBehaviour(() -> isSearchVisible())); return searchPanel; diff --git a/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/component/ObjectListPanel.java b/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/component/ObjectListPanel.java index c7bcb4d8ba2..ef07ce98928 100644 --- a/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/component/ObjectListPanel.java +++ b/gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/component/ObjectListPanel.java @@ -636,6 +636,14 @@ protected void searchPerformed(ObjectQuery query, AjaxRequestTarget target) { ObjectListPanel.this.searchPerformed(query, target); } + @Override + protected void saveSearch(Search search, AjaxRequestTarget target) { + PageStorage storage = getPageStorage(getStorageKey()); + if (storage != null) { + storage.setSearch(search); + } + } + }; return searchPanel; @@ -823,15 +831,7 @@ private void searchPerformed(ObjectQuery query, AjaxRequestTarget target) { // } provider.setQuery(customQuery); - String storageKey = getStorageKey(); - if (StringUtils.isNotEmpty(storageKey)) { - PageStorage storage = getPageStorage(storageKey); - if (storage != null) { - storage.setSearch(searchModel.getObject()); - storage.setPaging(null); - } - } - + saveSearchModel(null); Table table = getTable(); table.setCurrentPage(null); target.add((Component) table); diff --git a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchFormPanel.java b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchFormPanel.java index 43bb65470f5..2f03e5bada4 100644 --- a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchFormPanel.java +++ b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchFormPanel.java @@ -33,11 +33,17 @@ protected void initLayout() { searchForm.setOutputMarkupId(true); SearchPanel search = new SearchPanel(ID_SEARCH, getModel()) { + private static final long serialVersionUID = 1L; @Override public void searchPerformed(ObjectQuery query, AjaxRequestTarget target) { SearchFormPanel.this.searchPerformed(query, target); } + + @Override + protected void saveSearch(Search search, AjaxRequestTarget target) { + SearchFormPanel.this.saveSearch(search, target); + } }; searchForm.add(search); } @@ -45,4 +51,7 @@ public void searchPerformed(ObjectQuery query, AjaxRequestTarget target) { protected void searchPerformed(ObjectQuery query, AjaxRequestTarget target) { } + + protected void saveSearch(Search search, AjaxRequestTarget target) { + } } diff --git a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchPanel.java b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchPanel.java index de496e4e199..68bd83a0d6d 100644 --- a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchPanel.java +++ b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchPanel.java @@ -571,6 +571,7 @@ private void initPopover() { ListView properties = new ListView(ID_PROPERTIES, new PropertyModel<>(moreDialogModel, MoreDialogDto.F_PROPERTIES)) { + private static final long serialVersionUID = 1L; @Override protected void populateItem(final ListItem item) { @@ -740,6 +741,10 @@ void searchPerformed(AjaxRequestTarget target) { void refreshSearchForm(AjaxRequestTarget target) { target.add(get(ID_FORM), get(ID_POPOVER)); + saveSearch(getModelObject(), target); + } + + protected void saveSearch(Search search, AjaxRequestTarget target) { } public void searchPerformed(ObjectQuery query, AjaxRequestTarget target) { From 6c1c748ba1a038bdc327dcabd030af74c1666950 Mon Sep 17 00:00:00 2001 From: Tony Tkacik Date: Thu, 3 Sep 2020 16:05:21 +0200 Subject: [PATCH 03/12] Added AnyType to Axiom Signed-off-by: Tony Tkacik --- infra/axiom/src/main/resources/axiom-types.axiom | 1 + .../prism/impl/schema/axiom/AxiomEnabledSchemaRegistry.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/infra/axiom/src/main/resources/axiom-types.axiom b/infra/axiom/src/main/resources/axiom-types.axiom index e78be9ce099..4730c58dbf9 100644 --- a/infra/axiom/src/main/resources/axiom-types.axiom +++ b/infra/axiom/src/main/resources/axiom-types.axiom @@ -19,6 +19,7 @@ model axiom-base-types { type Integer; type Binary; type DateTime; + type AnyType; type QName { supertype Uri; diff --git a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/schema/axiom/AxiomEnabledSchemaRegistry.java b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/schema/axiom/AxiomEnabledSchemaRegistry.java index 16d3e88453d..402892cfec0 100644 --- a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/schema/axiom/AxiomEnabledSchemaRegistry.java +++ b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/schema/axiom/AxiomEnabledSchemaRegistry.java @@ -59,9 +59,12 @@ public class AxiomEnabledSchemaRegistry extends SchemaRegistryImpl { private static final BiMap AXIOM_XSD_TYPES = ImmutableBiMap.builder() .put(AxiomName.builtIn("String"), DOMUtil.XSD_STRING) + .put(AxiomName.builtIn("Boolean"), DOMUtil.XSD_BOOLEAN) .put(AxiomName.builtIn("DateTime"), DOMUtil.XSD_DATETIME) .put(AxiomName.builtIn("Uri"), DOMUtil.XSD_ANYURI) + .put(AxiomName.builtIn("Binary"), DOMUtil.XSD_BASE64BINARY) .put(AxiomName.builtIn("Integer"), DOMUtil.XSD_INT) + .put(AxiomName.builtIn("AnyType"), DOMUtil.XSD_ANYTYPE) .put(AxiomName.from(PRISM_NAMESPACE, "ItemPath"), new QName(PRISM_TYPES,"ItemPathType")) .build(); private static final AxiomName DISPLAY_NAME = PROPERTY_ITEM.localName("displayName"); From 7e94fb90f50ae167e2aac17449f94d94b9282cd1 Mon Sep 17 00:00:00 2001 From: kate Date: Thu, 3 Sep 2020 16:23:56 +0200 Subject: [PATCH 04/12] styles fixing --- .../web/component/search/ReferenceValueSearchPanel.html | 7 ++++--- .../src/main/resources/static/less/midpoint-theme.less | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/ReferenceValueSearchPanel.html b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/ReferenceValueSearchPanel.html index 6f031e6655d..90686719c4b 100644 --- a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/ReferenceValueSearchPanel.html +++ b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/ReferenceValueSearchPanel.html @@ -9,9 +9,10 @@
-
-
- +
+
+
diff --git a/gui/admin-gui/src/main/resources/static/less/midpoint-theme.less b/gui/admin-gui/src/main/resources/static/less/midpoint-theme.less index 931bbd91313..7ce671c8474 100644 --- a/gui/admin-gui/src/main/resources/static/less/midpoint-theme.less +++ b/gui/admin-gui/src/main/resources/static/less/midpoint-theme.less @@ -1657,6 +1657,7 @@ th.composited-icon { .search-item { display: inline-block; + min-width: 270px; border-color: #ddd; border-radius: 3px; background-color: #f4f4f4; From 6a48d387a06b4e7f3cccd515bedbb24448d1702e Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Thu, 3 Sep 2020 17:03:22 +0200 Subject: [PATCH 05/12] Add support for transformation metadata 1) Changed schema for transformation metadata and created built-in mappings for them. 2) Added support for transient prism values. 3) Added support for metadata item persistence. Also: 4) Clearing pre-existing metadata in asIs mappings. This resolves a bug when there was two sets of metadata from asIs mapping: with metadata enabled (having correct provenance) and without metadata enabled (having incorrect provenance). 5) Fixed RawType serialization (at couple of places). 6) Added per-item metadata configuration ($template/item/meta). --- .../prism/show/PreviewChangesTabPanel.java | 2 +- .../evolveum/midpoint/prism/PrismValue.java | 10 + .../midpoint/prism/SerializationOptions.java | 22 ++ .../prism/impl/PrismContainerValueImpl.java | 5 +- .../prism/impl/PrismPropertyValueImpl.java | 3 + .../prism/impl/PrismReferenceValueImpl.java | 3 + .../midpoint/prism/impl/PrismValueImpl.java | 15 +- .../prism/impl/marshaller/BeanMarshaller.java | 2 + .../impl/marshaller/PrismMarshaller.java | 72 ++++++- .../impl/xml/XmlTypeConverterInternal.java | 52 ++--- .../midpoint/schema/DeltaConvertor.java | 7 +- .../schema/MidPointPrismContextFactory.java | 2 +- .../xml/ns/public/common/common-core-3.xsd | 16 +- .../ns/public/common/common-metadata-3.axiom | 48 ++--- .../ns/public/common/common-metadata-3.xsd | 76 ++----- .../common/mapping/AbstractMappingImpl.java | 8 +- .../ItemValueMetadataProcessingSpec.java | 37 +++- .../MetadataItemProcessingSpecImpl.java | 30 ++- .../TransformationalMetadataComputation.java | 17 +- .../metadata/ValueMetadataComputation.java | 16 ++ .../builtin/BaseBuiltinMetadataMapping.java | 19 ++ .../builtin/ProvenanceBuiltinMapping.java | 15 -- .../builtin/TransformationBuiltinMapping.java | 133 ++++++++++++ .../model/intest/TestValueMetadata.java | 23 +- ...template-provenance-metadata-recording.xml | 47 ++-- .../test/resources/metadata/user-alice.xml | 201 +++++++++++------- .../AbstractExpressionEvaluator.java | 18 +- .../repo/sql/helpers/ObjectUpdater.java | 7 +- 28 files changed, 624 insertions(+), 282 deletions(-) create mode 100644 model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/TransformationBuiltinMapping.java diff --git a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/prism/show/PreviewChangesTabPanel.java b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/prism/show/PreviewChangesTabPanel.java index d4179acda1a..36e7cae3a72 100644 --- a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/prism/show/PreviewChangesTabPanel.java +++ b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/prism/show/PreviewChangesTabPanel.java @@ -85,7 +85,7 @@ private void initModels(){ if (modelContext != null) { if (modelContext.getFocusContext() != null) { addIgnoreNull(primaryDeltas, CloneUtil.clone(modelContext.getFocusContext().getPrimaryDelta())); - addIgnoreNull(secondaryDeltas, CloneUtil.clone(modelContext.getFocusContext().getSecondaryDelta())); + addIgnoreNull(secondaryDeltas, CloneUtil.clone(modelContext.getFocusContext().getSecondaryDelta())); // todo adapt this, see MID-6465 } for (ModelProjectionContext projCtx : modelContext.getProjectionContexts()) { addIgnoreNull(primaryDeltas, CloneUtil.clone(projCtx.getPrimaryDelta())); diff --git a/infra/prism-api/src/main/java/com/evolveum/midpoint/prism/PrismValue.java b/infra/prism-api/src/main/java/com/evolveum/midpoint/prism/PrismValue.java index 63210a795f4..0139c997b8f 100644 --- a/infra/prism-api/src/main/java/com/evolveum/midpoint/prism/PrismValue.java +++ b/infra/prism-api/src/main/java/com/evolveum/midpoint/prism/PrismValue.java @@ -228,4 +228,14 @@ default Object getRealValueOrRawType(PrismContext prismContext) { // Collection deltas, boolean ignoreMetadata, boolean isLiteral); Object find(ItemPath path); + + /** + * @return True if the value is transient, so it won't be serialized if serialization + * of transient value is disabled. + */ + @Experimental + boolean isTransient(); + + @Experimental + void setTransient(boolean value); } diff --git a/infra/prism-api/src/main/java/com/evolveum/midpoint/prism/SerializationOptions.java b/infra/prism-api/src/main/java/com/evolveum/midpoint/prism/SerializationOptions.java index d6d67635e68..2519d24e5ba 100644 --- a/infra/prism-api/src/main/java/com/evolveum/midpoint/prism/SerializationOptions.java +++ b/infra/prism-api/src/main/java/com/evolveum/midpoint/prism/SerializationOptions.java @@ -23,6 +23,11 @@ public class SerializationOptions implements Cloneable { */ private boolean skipIndexOnly; + /** + * Should we skip values marked as transient? + */ + private boolean skipTransient; + private ItemNameQualificationStrategy itemNameQualificationStrategy; /** @@ -165,6 +170,23 @@ public boolean isSkipIndexOnly() { return skipIndexOnly; } + public void setSkipTransient(boolean skipTransient) { + this.skipTransient = skipTransient; + } + + public SerializationOptions skipTransient(boolean value) { + setSkipTransient(value); + return this; + } + + public static SerializationOptions createSkipTransient() { + return new SerializationOptions().skipTransient(true); + } + + public boolean isSkipTransient() { + return skipTransient; + } + // public ItemNameQualificationStrategy getItemNameQualificationStrategy() { // return itemNameQualificationStrategy; // } diff --git a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismContainerValueImpl.java b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismContainerValueImpl.java index c88bb82c400..9dace835cc8 100644 --- a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismContainerValueImpl.java +++ b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismContainerValueImpl.java @@ -1399,7 +1399,7 @@ public boolean equivalent(PrismContainerValue other) { // TODO consider taking equivalence strategy into account @Override - public int hashCode(ParameterizedEquivalenceStrategy strategy) { + public int hashCode(@NotNull ParameterizedEquivalenceStrategy strategy) { final int prime = 31; int result = super.hashCode(strategy); // Do not include id. containers with non-null id and null id may still be considered equivalent @@ -1420,6 +1420,9 @@ public String toString() { sb.append(getId()); sb.append("):"); sb.append(getItems()); + if (isTransient()) { + sb.append(", transient"); + } return sb.toString(); } diff --git a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismPropertyValueImpl.java b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismPropertyValueImpl.java index 0fc8c00e8a7..865afb059ec 100644 --- a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismPropertyValueImpl.java +++ b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismPropertyValueImpl.java @@ -565,6 +565,9 @@ private void dumpSuffix(StringBuilder builder) { builder.append(", expression: "); builder.append(getExpression()); } + if (isTransient()) { + builder.append(", transient"); + } } @Override diff --git a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismReferenceValueImpl.java b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismReferenceValueImpl.java index 6dba5b1044b..90ce45f884b 100644 --- a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismReferenceValueImpl.java +++ b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismReferenceValueImpl.java @@ -543,6 +543,9 @@ public String toString() { if (referentialIntegrity != null) { sb.append(", RI=").append(referentialIntegrity); } + if (isTransient()) { + sb.append(", transient"); + } sb.append(")"); return sb.toString(); } diff --git a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismValueImpl.java b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismValueImpl.java index b127cdeefb6..55090cb982a 100644 --- a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismValueImpl.java +++ b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/PrismValueImpl.java @@ -44,6 +44,8 @@ public abstract class PrismValueImpl extends AbstractFreezable implements PrismV protected transient PrismContext prismContext; + private boolean isTransient; + PrismValueImpl() { } @@ -249,9 +251,10 @@ protected void copyValues(CloneStrategy strategy, PrismValueImpl clone) { clone.prismContext = this.prismContext; } clone.valueMetadata = valueMetadata != null ? valueMetadata.clone() : null; + clone.isTransient = isTransient; } - EquivalenceStrategy getEqualsHashCodeStrategy() { + private EquivalenceStrategy getEqualsHashCodeStrategy() { return defaultIfNull(defaultEquivalenceStrategy, EquivalenceStrategy.NOT_LITERAL); } @@ -449,4 +452,14 @@ protected void performFreeze() { } super.performFreeze(); } + + @Override + public boolean isTransient() { + return isTransient; + } + + @Override + public void setTransient(boolean value) { + isTransient = value; + } } diff --git a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/BeanMarshaller.java b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/BeanMarshaller.java index 0ab92b39e31..8300434e771 100644 --- a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/BeanMarshaller.java +++ b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/BeanMarshaller.java @@ -116,6 +116,8 @@ public XNodeImpl marshall(@Nullable Object inputBean, @Nullable SerializationCon xnode.setTypeQName(ObjectUtils.defaultIfNull(prismContext.getDefaultReferenceTypeName(), ObjectReferenceType.COMPLEX_TYPE)); xnode.setExplicitTypeDeclaration(true); // probably not much correct, but... return xnode; + } else if (bean instanceof RawType && ((RawType) bean).getXnode() != null) { + return (XNodeImpl) ((RawType) bean).getXnode(); } else { return marshalToPrimitive(bean, ctx); } diff --git a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismMarshaller.java b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismMarshaller.java index b1c3b1c209d..2a60995d1f3 100644 --- a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismMarshaller.java +++ b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismMarshaller.java @@ -15,15 +15,18 @@ import com.evolveum.midpoint.prism.xml.XmlTypeConverter; import com.evolveum.midpoint.prism.xnode.MapXNode; import com.evolveum.midpoint.prism.xnode.MetadataAware; +import com.evolveum.midpoint.prism.xnode.XNode; import com.evolveum.midpoint.util.DOMUtil; import com.evolveum.midpoint.util.JAXBUtil; import com.evolveum.midpoint.util.QNameUtil; +import com.evolveum.midpoint.util.annotation.Experimental; import com.evolveum.midpoint.util.exception.SchemaException; import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; import com.evolveum.prism.xml.ns._public.query_3.SearchFilterType; import com.evolveum.prism.xml.ns._public.types_3.EvaluationTimeType; import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; +import com.evolveum.prism.xml.ns._public.types_3.RawType; import com.evolveum.prism.xml.ns._public.types_3.ReferentialIntegrityType; import org.apache.commons.collections4.CollectionUtils; @@ -168,16 +171,18 @@ RootXNodeImpl marshalAnyData(@NotNull Object object, QName itemName, ItemDefinit */ @NotNull private XNodeImpl marshalItemContent(@NotNull Item item, - ItemDefinition itemDefinition, SerializationContext context, + ItemDefinition itemDefinition, SerializationContext ctx, Collection itemsToSkip) throws SchemaException { - if (item.isEmpty() && item.isIncomplete()) { + + List valuesToMarshal = getValuesToMarshal(item, ctx); + if (valuesToMarshal.isEmpty() && item.isIncomplete()) { return new IncompleteMarkerXNodeImpl(); - } else if (item.size() == 1 && !item.isIncomplete()) { - return marshalItemValue(item.getAnyValue(), itemDefinition, null, context, itemsToSkip); + } else if (valuesToMarshal.size() == 1 && !item.isIncomplete()) { + return marshalItemValue(valuesToMarshal.get(0), itemDefinition, null, ctx, itemsToSkip); } else { ListXNodeImpl xlist = new ListXNodeImpl(); - for (PrismValue val : item.getValues()) { - xlist.add(marshalItemValue(val, itemDefinition, null, context, itemsToSkip)); + for (PrismValue val : valuesToMarshal) { + xlist.add(marshalItemValue(val, itemDefinition, null, ctx, itemsToSkip)); } if (item.isIncomplete()) { xlist.add(new IncompleteMarkerXNodeImpl()); @@ -186,6 +191,20 @@ private XNodeImpl marshalItemContent(@NotNull Item item, } } + private List getValuesToMarshal(Item item, SerializationContext ctx) { + if (ctx == null || ctx.getOptions() == null || !ctx.getOptions().isSkipTransient()) { + return item.getValues(); + } else { + List valuesToMarshal = new ArrayList<>(); + for (PrismValue value : item.getValues()) { + if (value == null || !value.isTransient()) { + valuesToMarshal.add(value); + } + } + return valuesToMarshal; + } + } + @NotNull private MapXNodeImpl marshalObjectContent(@NotNull PrismObject object, @NotNull PrismObjectDefinition objectDefinition, SerializationContext ctx) throws SchemaException { MapXNodeImpl xmap = new MapXNodeImpl(); @@ -325,7 +344,7 @@ private void marshalContainerValue(MapXNodeImpl xmap, PrismContainerValue con for (ItemDefinition itemDef: containerDefinition.getDefinitions()) { ItemName elementName = itemDef.getItemName(); Item item = containerVal.findItem(elementName); - if (item != null) { + if (item != null && !skipBecauseTransient(item, ctx)) { XNodeImpl xsubnode; if (shouldSkipItem(itemsToSkip, elementName, itemDef, ctx) && !item.hasNoValues()) { xsubnode = new IncompleteMarkerXNodeImpl(); @@ -341,7 +360,7 @@ private void marshalContainerValue(MapXNodeImpl xmap, PrismContainerValue con // E.g. in run-time schema. Therefore we must also iterate over items and not just item definitions. for (Item item : containerVal.getItems()) { QName elementName = item.getElementName(); - if (!marshaledItems.contains(elementName)) { + if (!marshaledItems.contains(elementName) && !skipBecauseTransient(item, ctx)) { XNodeImpl xsubnode; if (shouldSkipItem(itemsToSkip, elementName, item.getDefinition(), ctx) && !item.hasNoValues()) { xsubnode = new IncompleteMarkerXNodeImpl(); @@ -353,6 +372,26 @@ private void marshalContainerValue(MapXNodeImpl xmap, PrismContainerValue con } } + @Experimental + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + private boolean skipBecauseTransient(Item item, SerializationContext ctx) { + if (ctx == null || ctx.getOptions() == null || !ctx.getOptions().isSkipTransient()) { + return false; + } + if (item.isIncomplete()) { + return false; // we need to store the "incomplete" flag + } + if (item.hasNoValues()) { + return false; // backwards compatibility (rather questionable) + } + for (PrismValue value : item.getValues()) { + if (!value.isTransient()) { + return false; + } + } + return true; + } + private boolean shouldSkipItem(Collection itemsToSkip, QName elementName, ItemDefinition itemDef, SerializationContext ctx) { return QNameUtil.contains(itemsToSkip, elementName) || @@ -476,7 +515,16 @@ private XNodeImpl serializePropertyValue(@NotNull PrismPropertyValue valu return createExpressionXNode(expression); } T realValue = value.getValue(); - if (realValue instanceof PolyString) { + if (realValue instanceof RawType) { + RawType rawValue = (RawType) realValue; + if (rawValue.isParsed()) { + return marshalItemValue(rawValue.getAlreadyParsedValue(), definition, typeNameIfNoDefinition, ctx, emptySet()); + } else if (rawValue.getXnode() != null) { + return (XNodeImpl) ((RawType) realValue).getXnode(); + } else { + throw new SchemaException("Cannot marshall empty RawType: " + value); + } + } else if (realValue instanceof PolyString) { return beanMarshaller.marshalPolyString((PolyString) realValue); } else if (realValue instanceof PolyStringType) { // should not occur ... return beanMarshaller.marshalPolyString(((PolyStringType) realValue).toPolyString()); @@ -522,6 +570,12 @@ private XNodeImpl serializePropertyRawValue(PrismPropertyValue value) { } T realValue = value.getValue(); if (realValue != null) { + if (realValue instanceof RawType) { + XNode xnode = ((RawType) realValue).getXnode(); + if (xnode != null) { + return (XNodeImpl) xnode; + } + } return createPrimitiveXNode(realValue, null); } else { throw new IllegalStateException("Neither real nor raw value present in " + value); diff --git a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/xml/XmlTypeConverterInternal.java b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/xml/XmlTypeConverterInternal.java index 0f7bdff0058..de06111527c 100644 --- a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/xml/XmlTypeConverterInternal.java +++ b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/xml/XmlTypeConverterInternal.java @@ -7,6 +7,19 @@ package com.evolveum.midpoint.prism.impl.xml; +import java.io.File; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.ZonedDateTime; +import java.util.GregorianCalendar; +import javax.xml.datatype.Duration; +import javax.xml.datatype.XMLGregorianCalendar; +import javax.xml.namespace.QName; + +import org.apache.commons.codec.binary.Base64; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + import com.evolveum.midpoint.prism.PrismConstants; import com.evolveum.midpoint.prism.impl.marshaller.ItemPathSerializerTemp; import com.evolveum.midpoint.prism.path.ItemPath; @@ -16,47 +29,14 @@ import com.evolveum.midpoint.prism.xml.XsdTypeMapper; import com.evolveum.midpoint.util.DOMUtil; import com.evolveum.midpoint.util.exception.SchemaException; -import org.apache.commons.codec.binary.Base64; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import javax.xml.datatype.DatatypeConfigurationException; -import javax.xml.datatype.DatatypeFactory; -import javax.xml.datatype.Duration; -import javax.xml.datatype.XMLGregorianCalendar; -import javax.xml.namespace.QName; -import java.io.File; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.time.ZonedDateTime; -import java.util.GregorianCalendar; /** * This is to be cleaned up later. Ideally, this functionality should be accessed by prism serializer interface. */ public class XmlTypeConverterInternal { - private static DatatypeFactory datatypeFactory = null; - - private static DatatypeFactory getDatatypeFactory() { - if (datatypeFactory == null) { - try { - datatypeFactory = DatatypeFactory.newInstance(); - } catch (DatatypeConfigurationException ex) { - throw new IllegalStateException("Cannot construct DatatypeFactory: " + ex.getMessage(), ex); - } - } - return datatypeFactory; - } - - /** - * @param val - * @param elementName - * @param doc - * @param recordType * @return created element - * @throws SchemaException */ public static Element toXsdElement(Object val, QName elementName, Document doc, boolean recordType) throws SchemaException { if (val == null) { @@ -133,9 +113,9 @@ public static String toXmlTextContent(Object val, QName elementName) { return XsdTypeMapper.BOOLEAN_XML_VALUE_FALSE; } } else if (type.equals(BigInteger.class)) { - return ((BigInteger) val).toString(); + return val.toString(); } else if (type.equals(BigDecimal.class)) { - return ((BigDecimal) val).toString(); + return val.toString(); } else if (type.equals(GregorianCalendar.class)) { XMLGregorianCalendar xmlCal = XmlTypeConverter.createXMLGregorianCalendar((GregorianCalendar) val); return xmlCal.toXMLFormat(); @@ -146,7 +126,7 @@ public static String toXmlTextContent(Object val, QName elementName) { } else if (XMLGregorianCalendar.class.isAssignableFrom(type)) { return ((XMLGregorianCalendar) val).toXMLFormat(); } else if (Duration.class.isAssignableFrom(type)) { - return ((Duration) val).toString(); + return val.toString(); } else if (type.equals(UniformItemPath.class) || type.equals(ItemPath.class)) { return ItemPathSerializerTemp.serializeWithDeclarations((ItemPath) val); } else { diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/DeltaConvertor.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/DeltaConvertor.java index 1c2fe9f7db4..ec59c71e7a3 100644 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/DeltaConvertor.java +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/DeltaConvertor.java @@ -222,14 +222,11 @@ public static ObjectDeltaType toObjectDeltaType(ObjectDelta delta) throws SchemaException, JAXBException { - return toObjectDeltaTypeXml(delta, null); - } - - public static String toObjectDeltaTypeXml(ObjectDelta delta, DeltaConversionOptions options) throws SchemaException, JAXBException { + public static String toObjectDeltaTypeXml(ObjectDelta delta, DeltaConversionOptions options) throws SchemaException { Validate.notNull(delta.getPrismContext(), "ObjectDelta without prismContext cannot be converted to XML"); ObjectDeltaType objectDeltaType = toObjectDeltaType(delta, options); SerializationOptions serializationOptions = new SerializationOptions() + .skipTransient(true) .serializeReferenceNames(DeltaConversionOptions.isSerializeReferenceNames(options)) .escapeInvalidCharacters(DeltaConversionOptions.isEscapeInvalidCharacters(options)); return delta.getPrismContext().xmlSerializer().options(serializationOptions) diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/MidPointPrismContextFactory.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/MidPointPrismContextFactory.java index 7863b34e60d..19050f6bcb1 100644 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/MidPointPrismContextFactory.java +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/MidPointPrismContextFactory.java @@ -107,7 +107,7 @@ private SchemaRegistryImpl createSchemaRegistry() throws SchemaException, IOExce } private void registerAxiomSchemas(SchemaRegistryImpl schemaRegistry) { - if(schemaRegistry instanceof AxiomEnabledSchemaRegistry) { + if (schemaRegistry instanceof AxiomEnabledSchemaRegistry) { AxiomEnabledSchemaRegistry axiomRegistry = (AxiomEnabledSchemaRegistry) schemaRegistry; AxiomModelStatementSource commonMetadata; try { diff --git a/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd b/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd index ff726fcf7eb..2eb5297c3d9 100755 --- a/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd +++ b/infra/schema/src/main/resources/xml/ns/public/common/common-core-3.xsd @@ -8017,7 +8017,7 @@ - + Whether values of this item should be persisted. @@ -8033,7 +8033,7 @@ - + Whether or how data should be persisted. @@ -8084,6 +8084,7 @@ Data are held only during computation. They are not provided to the client. + (Not implemented yet.) @@ -15034,6 +15035,17 @@ + + + + Definition of handling of a metadata items for this particular item. + + + true + 4.2 + + + diff --git a/infra/schema/src/main/resources/xml/ns/public/common/common-metadata-3.axiom b/infra/schema/src/main/resources/xml/ns/public/common/common-metadata-3.axiom index af4f73707ac..04b8c5e3ed9 100644 --- a/infra/schema/src/main/resources/xml/ns/public/common/common-metadata-3.axiom +++ b/infra/schema/src/main/resources/xml/ns/public/common/common-metadata-3.axiom @@ -164,46 +164,36 @@ prism:model common-metadata { container TransformationMetadataType { displayName "Transformation Metadata"; operational true; - container source { - type ValueSourceType; - documentation "Source(s) of the value"; - maxOccurs "unbounded"; - } - container transformer { - type ValueTransformerType; - documentation "Transformer(s) that acted upon the value."; + container mappingTransformation { + type MappingTransformationType; + documentation "Mapping transformation that led to the current value."; maxOccurs "unbounded"; } } - container ValueSourceType { - property kind { - type Uri; - } - property name { - type String; - } - reference objectRef { - documentation """ - A midPoint object representing the source of the value (resource object, - object template, assignment holder, user, ...) - """; + container MappingTransformationType { + documentation "Informs that the value was provided by a mapping (and how exactly)."; + displayName "Transformation Metadata"; + operational true; + container mappingSpecification { + type MappingSpecificationType; + documentation "Specification of the mapping."; } - property storage { - type prism:ItemPath; - documentation "Where was the original value stored."; + container source { + type MappingSourceType; + documentation "Sources for the mapping."; + maxOccurs "unbounded"; } } - container ValueTransformerType { - property kind { - type Uri; - } + container MappingSourceType { property name { type String; + documentation "Name of the mapping source."; } - reference objectRef { - documentation "A midPoint object holding the transformer"; + property value { + type AnyType; + documentation "Source value itself. (Along with its own metadata.)"; } } diff --git a/infra/schema/src/main/resources/xml/ns/public/common/common-metadata-3.xsd b/infra/schema/src/main/resources/xml/ns/public/common/common-metadata-3.xsd index 6dbf1fa72f9..632c2f38538 100644 --- a/infra/schema/src/main/resources/xml/ns/public/common/common-metadata-3.xsd +++ b/infra/schema/src/main/resources/xml/ns/public/common/common-metadata-3.xsd @@ -489,17 +489,10 @@ - + - Source(s) of the value. - - - - - - - Transformer(s) that acted upon the value. + Mapping transformation that led to the current value. @@ -507,100 +500,65 @@ - + - Specifies a value source. + Informs that the value was provided by a mapping (and how exactly). true + 4.2 - - - - Type of the source (resource, object template, assignment, user action, ...) - - - - - - - Name of the source (whatever that means). - - - - + - A midPoint object representing the source of the value (resource object, object template, assignment holder, user, ...) + Specification of the mapping. - + - Specific information needed to identify the source e.g. within the midPoint object - (mapping name, assignment path, ...). - - - - - - - Where was the original value stored. + Sources for the mapping. - + - + - Specifies agent that transformed the value. true + 4.2 + Value source - - - - Type of the transformer. Currently the only supported one is "mapping". - - - - Name of the transformer. Currently mapping name. - - - - - - - A midPoint object holding the transformer. + Name of the mapping source. - + - Specific information needed to identify the transformer e.g. within the midPoint object - (mapping name, assignment path, ...). + Source value itself. (Along with its own metadata.) - + diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/AbstractMappingImpl.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/AbstractMappingImpl.java index f21e6fe5bc0..d30570b5047 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/AbstractMappingImpl.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/AbstractMappingImpl.java @@ -266,7 +266,7 @@ public abstract class AbstractMappingImpl> sources = new ArrayList<>(); + @NotNull final List> sources = new ArrayList<>(); /** * State of the mapping evaluation. @@ -1502,4 +1502,10 @@ public ModelCommonBeans getBeans() { public @NotNull MappingSpecificationType getMappingSpecification() { return mappingSpecification; } + + public List getSourceNames() { + return sources.stream() + .map(Source::getName) + .collect(Collectors.toList()); + } } diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/ItemValueMetadataProcessingSpec.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/ItemValueMetadataProcessingSpec.java index 6b0f6ea7442..8aeb364389d 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/ItemValueMetadataProcessingSpec.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/ItemValueMetadataProcessingSpec.java @@ -16,6 +16,7 @@ import com.evolveum.midpoint.model.common.util.ObjectTemplateIncludeProcessor; import com.evolveum.midpoint.prism.path.ItemPath; import com.evolveum.midpoint.prism.path.ItemPathCollectionsUtil; +import com.evolveum.midpoint.prism.path.PathSet; import com.evolveum.midpoint.repo.common.ObjectResolver; import com.evolveum.midpoint.schema.result.OperationResult; import com.evolveum.midpoint.task.api.Task; @@ -117,15 +118,25 @@ private void addFromObjectTemplate(ObjectTemplateType rootTemplate, ItemPath dat private void addFromObjectTemplate(ObjectTemplateType template, ItemPath dataPath) { try { - if (isHandlingApplicable(template.getMeta(), dataPath)) { - addMetadataMappings(template.getMeta().getMapping()); - addMetadataItems(template.getMeta().getItem(), dataPath); + addHandling(template.getMeta(), dataPath); + + for (ObjectTemplateItemDefinitionType itemDef : template.getItem()) { + if (itemDef.getRef() != null && itemDef.getRef().getItemPath().equivalent(dataPath)) { + addHandling(itemDef.getMeta(), dataPath); + } } } catch (SchemaException e) { throw new TunnelException(e); } } + private void addHandling(MetadataHandlingType handling, ItemPath dataPath) throws SchemaException { + if (isHandlingApplicable(handling, dataPath)) { + addMetadataMappings(handling.getMapping()); + addMetadataItems(handling.getItem(), dataPath); + } + } + private boolean isHandlingApplicable(MetadataHandlingType handling, ItemPath dataPath) throws SchemaException { return handling != null && ProcessingUtil.doesApplicabilityMatch(handling.getApplicability(), dataPath); } @@ -219,6 +230,26 @@ public boolean isFullProcessing(ItemPath itemPath) throws SchemaException { return getProcessing(itemPath) == ItemProcessingType.FULL; } + public Collection getTransientPaths() { + PathSet transientPaths = new PathSet(); + PathSet persistentPaths = new PathSet(); + for (MetadataItemDefinitionType item : itemDefinitions) { + ItemPersistenceType persistence = item.getPersistence(); + if (persistence != null) { + ItemPath path = item.getRef().getItemPath(); + if (persistence == ItemPersistenceType.PERSISTENT) { + persistentPaths.add(path); + } else { + transientPaths.add(path); + } + } + } + for (ItemPath persistentPath : persistentPaths) { + transientPaths.remove(persistentPath); + } + return transientPaths; + } + // For item-contained mappings the target exclusion can be checked twice private boolean isMetadataMappingApplicable(MetadataMappingType mapping) { return hasCompatibleScope(mapping) && diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/MetadataItemProcessingSpecImpl.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/MetadataItemProcessingSpecImpl.java index 518d9b4d91d..2537fd29e6c 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/MetadataItemProcessingSpecImpl.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/MetadataItemProcessingSpecImpl.java @@ -11,18 +11,18 @@ import java.util.Arrays; import java.util.List; -import com.evolveum.midpoint.model.api.MetadataItemProcessingSpec; - -import com.evolveum.midpoint.util.DebugDumpable; -import com.evolveum.midpoint.util.DebugUtil; +import com.evolveum.prism.xml.ns._public.types_3.ItemPathType; import org.jetbrains.annotations.NotNull; +import com.evolveum.midpoint.model.api.MetadataItemProcessingSpec; import com.evolveum.midpoint.model.common.util.ObjectTemplateIncludeProcessor; import com.evolveum.midpoint.prism.path.ItemPath; import com.evolveum.midpoint.repo.common.ObjectResolver; import com.evolveum.midpoint.schema.result.OperationResult; import com.evolveum.midpoint.task.api.Task; +import com.evolveum.midpoint.util.DebugDumpable; +import com.evolveum.midpoint.util.DebugUtil; import com.evolveum.midpoint.util.MiscUtil; import com.evolveum.midpoint.util.exception.*; import com.evolveum.midpoint.util.logging.Trace; @@ -92,14 +92,20 @@ private void addFromObjectTemplate(ObjectTemplateType rootTemplate, ObjectResolv } private void addFromObjectTemplate(ObjectTemplateType template) { + addMetadataHandling(template.getMeta(), null); + template.getItem().forEach( + itemDef -> addMetadataHandling(itemDef.getMeta(), itemDef.getRef())); + } + + private void addMetadataHandling(MetadataHandlingType metadataHandling, ItemPathType dataItemPathBean) { try { - MetadataHandlingType metadataHandling = template.getMeta(); if (metadataHandling != null) { for (MetadataItemDefinitionType metadataItemDef : metadataHandling.getItem()) { if (metadataItemDef.getRef() != null && metadataItemDef.getRef().getItemPath().equivalent(metadataItemPath)) { ItemProcessingType processing = ProcessingUtil.getProcessingOfItem(metadataItemDef); if (processing != null) { - processingSpecs.add(new ProcessingSpec(processing, metadataHandling.getApplicability(), metadataItemDef.getApplicability())); + processingSpecs.add(new ProcessingSpec(processing, dataItemPathBean, + metadataHandling.getApplicability(), metadataItemDef.getApplicability())); } } } @@ -119,17 +125,22 @@ public String debugDump(int indent) { private static class ProcessingSpec implements DebugDumpable { private final ItemProcessingType processing; + private final ItemPath dataItemPath; private final List applicabilityList; - private ProcessingSpec(ItemProcessingType processing, + private ProcessingSpec(ItemProcessingType processing, ItemPathType dataItemPathBean, MetadataProcessingApplicabilitySpecificationType... applicabilityList) { this.processing = processing; + this.dataItemPath = dataItemPathBean != null ? dataItemPathBean.getItemPath() : null; this.applicabilityList = Arrays.asList(applicabilityList); } - public boolean appliesTo(ItemPath dataItem) throws SchemaException { + public boolean appliesTo(ItemPath dataItemPath) throws SchemaException { + if (this.dataItemPath != null && !this.dataItemPath.equivalent(dataItemPath)) { + return false; + } for (MetadataProcessingApplicabilitySpecificationType applicability : applicabilityList) { - if (!ProcessingUtil.doesApplicabilityMatch(applicability, dataItem)) { + if (!ProcessingUtil.doesApplicabilityMatch(applicability, dataItemPath)) { return false; } } @@ -140,6 +151,7 @@ public boolean appliesTo(ItemPath dataItem) throws SchemaException { public String debugDump(int indent) { StringBuilder sb = new StringBuilder(); DebugUtil.debugDumpWithLabelLn(sb, "processing", String.valueOf(processing), indent); + DebugUtil.debugDumpWithLabelLn(sb, "dataItemPath", String.valueOf(dataItemPath), indent); DebugUtil.debugDumpWithLabelLn(sb, "applicabilityList", applicabilityList, indent); return sb.toString(); } diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/TransformationalMetadataComputation.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/TransformationalMetadataComputation.java index 403b9a72360..b28613c041d 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/TransformationalMetadataComputation.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/TransformationalMetadataComputation.java @@ -8,6 +8,7 @@ package com.evolveum.midpoint.model.common.mapping.metadata; import com.evolveum.midpoint.model.common.ModelCommonBeans; +import com.evolveum.midpoint.model.common.mapping.AbstractMappingImpl; import com.evolveum.midpoint.model.common.mapping.MappingEvaluationEnvironment; import com.evolveum.midpoint.model.common.mapping.MappingImpl; import com.evolveum.midpoint.model.common.mapping.metadata.builtin.BuiltinMetadataMapping; @@ -42,14 +43,20 @@ public class TransformationalMetadataComputation extends ValueMetadataComputatio */ @NotNull private final List inputValues; + /** + * Mapping in context of which this transformation is being executed. + */ + @NotNull private final AbstractMappingImpl mapping; + private TransformationalMetadataComputation( @NotNull List inputValues, @NotNull ItemValueMetadataProcessingSpec processingSpec, - @Nullable MappingSpecificationType mappingSpecification, + @NotNull AbstractMappingImpl mapping, @NotNull ModelCommonBeans beans, MappingEvaluationEnvironment env) { - super(processingSpec, mappingSpecification, beans, env); + super(processingSpec, mapping.getMappingSpecification(), beans, env); this.inputValues = inputValues; + this.mapping = mapping; } @Override @@ -67,7 +74,7 @@ public static TransformationalMetadataComputation forMapping(List in MappingEvaluationEnvironment env = new MappingEvaluationEnvironment( "metadata evaluation in " + mapping.getMappingContextDescription(), mapping.getNow(), mapping.getTask()); - return new TransformationalMetadataComputation(inputValues, processingSpec, mapping.getMappingSpecification(), mapping.getBeans(), env); + return new TransformationalMetadataComputation(inputValues, processingSpec, mapping, mapping.getBeans(), env); } @Override @@ -91,6 +98,10 @@ public List getInputValues() { return Collections.unmodifiableList(inputValues); } + public List getSourceNames() { + return mapping.getSourceNames(); + } + @Override void createCustomMappingVariables(MetadataMappingBuilder builder, MetadataMappingType metadataMappingBean) { super.createCustomMappingVariables(builder, metadataMappingBean); diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/ValueMetadataComputation.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/ValueMetadataComputation.java index ab2fda66cac..a0ef01fd133 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/ValueMetadataComputation.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/ValueMetadataComputation.java @@ -107,6 +107,7 @@ public ValueMetadataType execute(OperationResult parentResult) throws Communicat processCustomMappings(); processBuiltinMappings(); recordOutput(); + applyPersistence(); return outputMetadata.asContainerable(); } catch (Throwable t) { result.recordFatalError(t); @@ -116,6 +117,21 @@ public ValueMetadataType execute(OperationResult parentResult) throws Communicat } } + private void applyPersistence() { + for (ItemPath transientPath : processingSpec.getTransientPaths()) { + markValuesTransient(transientPath); + } + } + + // We assume there is only a single item corresponding to given path + private void markValuesTransient(ItemPath path) { + Item item = outputMetadata.findItem(path); + if (item != null) { + LOGGER.trace("Marking {} values of {} as transient", item.size(), path); + item.getValues().forEach(value -> value.setTransient(true)); + } + } + private void recordOutput() { result.addReturn("summary", outputMetadata.toString()); // temporary } diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/BaseBuiltinMetadataMapping.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/BaseBuiltinMetadataMapping.java index 2ab622e669c..482b3e1e75c 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/BaseBuiltinMetadataMapping.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/BaseBuiltinMetadataMapping.java @@ -9,6 +9,8 @@ import javax.annotation.PostConstruct; +import com.evolveum.midpoint.prism.PrismValue; + import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; @@ -18,6 +20,8 @@ import com.evolveum.midpoint.prism.path.ItemPath; import com.evolveum.midpoint.util.exception.SchemaException; +import java.util.List; + /** * TODO */ @@ -50,4 +54,19 @@ void addPropertyRealValue(PrismContainerValue outputMetadata, Object value) t property.addRealValue(value); } } + + String dumpInput(List inputValues) { + if (inputValues.isEmpty()) { + return " (no values)"; + } else { + StringBuilder sb = new StringBuilder(); + for (PrismValue inputValue : inputValues) { + if (inputValue != null) { + sb.append(" - ").append(inputValue.toString()).append(" with metadata:\n"); + sb.append(inputValue.getValueMetadata().debugDump(2)).append("\n"); + } + } + return sb.toString(); + } + } } diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/ProvenanceBuiltinMapping.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/ProvenanceBuiltinMapping.java index 46f84436f49..f40cdfae094 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/ProvenanceBuiltinMapping.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/ProvenanceBuiltinMapping.java @@ -179,21 +179,6 @@ private ProvenanceMetadataType compute() { } } - private String dumpInput(List inputValues) { - if (inputValues.isEmpty()) { - return " (no values)"; - } else { - StringBuilder sb = new StringBuilder(); - for (PrismValue inputValue : inputValues) { - if (inputValue != null) { - sb.append(" - ").append(inputValue.toString()).append(" with metadata:\n"); - sb.append(inputValue.getValueMetadata().debugDump(2)).append("\n"); - } - } - return sb.toString(); - } - } - // private String dumpInputMetadata(List values) { // if (values.isEmpty()) { // return " (no values)"; diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/TransformationBuiltinMapping.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/TransformationBuiltinMapping.java new file mode 100644 index 00000000000..419fbf48ac3 --- /dev/null +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/mapping/metadata/builtin/TransformationBuiltinMapping.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.model.common.mapping.metadata.builtin; + +import static com.evolveum.midpoint.util.DebugUtil.lazy; + +import java.util.List; + +import org.jetbrains.annotations.NotNull; +import org.springframework.stereotype.Component; + +import com.evolveum.midpoint.model.common.mapping.metadata.ConsolidationMetadataComputation; +import com.evolveum.midpoint.model.common.mapping.metadata.TransformationalMetadataComputation; +import com.evolveum.midpoint.prism.PrismValue; +import com.evolveum.midpoint.prism.path.ItemPath; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.xml.ns._public.common.common_3.MappingSourceType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.MappingTransformationType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.TransformationMetadataType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ValueMetadataType; +import com.evolveum.prism.xml.ns._public.types_3.RawType; + +import javax.xml.namespace.QName; + +/** + * Mapping that manages transformation metadata. + */ +@Component +public class TransformationBuiltinMapping extends BaseBuiltinMetadataMapping { + + private static final Trace LOGGER = TraceManager.getTrace(TransformationBuiltinMapping.class); + + private static final ItemPath PATH = ItemPath.create(ValueMetadataType.F_TRANSFORMATION); + + @SuppressWarnings("unused") + TransformationBuiltinMapping() { + super(PATH); + } + + /** + * Transformation merges the acquisitions into single yield. + */ + @Override + public void applyForTransformation(@NotNull TransformationalMetadataComputation computation) { + List input = computation.getInputValues(); + + LOGGER.trace("Computing transformation metadata during value transformation. Input values:\n{}", + lazy(() -> dumpInput(input))); + + MappingTransformationType mappingTransformation = new MappingTransformationType(prismContext) + .mappingSpecification(computation.getMappingSpecification()); + + List sourceNames = computation.getSourceNames(); + if (computation.getInputValues().size() < sourceNames.size()) { + throw new IllegalStateException("Couldn't compute transformational metadata: there are less values (" + + computation.getInputValues().size() + ") than sources (" + sourceNames.size() + ") in " + + computation.getContextDescription()); + } + + for (int i = 0; i < sourceNames.size(); i++) { + MappingSourceType source = new MappingSourceType(prismContext); + + QName sourceName = sourceNames.get(i); + if (sourceName != null) { + source.setName(sourceName.getLocalPart()); + } + + PrismValue inputValue = computation.getInputValues().get(i); + if (inputValue != null) { + PrismValue inputValueClone = inputValue.clone(); + markNotTransient(inputValueClone); + source.setValue(new RawType(inputValueClone, null, prismContext)); + } + + mappingTransformation.getSource().add(source); + } + + TransformationMetadataType transformation = new TransformationMetadataType(prismContext) + .mappingTransformation(mappingTransformation); + + LOGGER.trace("Output: transformation:\n{}", lazy(() -> transformation.asPrismContainerValue().debugDump())); + + computation.getOutputMetadataValueBean().setTransformation(transformation); + } + + /** + * The value can contain transient values e.g. because during its computation the transformation metadata was + * marked as transient. But there can be situations when transformation metadata for the current value are + * non-transient. Then we could want to store them completely, i.e. with transformation metadata of source values. + * So we must mark them as not transient. + */ + private void markNotTransient(PrismValue rootValue) { + rootValue.accept(visitable -> { + if (visitable instanceof PrismValue) { + PrismValue value = (PrismValue) visitable; + value.setTransient(false); + value.getValueMetadataAsContainer().valuesStream() + .forEach(this::markNotTransient); + } + }); + } + + /** + * During consolidation we simply take (any) value. + */ + @Override + public void applyForConsolidation(@NotNull ConsolidationMetadataComputation computation) { + TransformationMetadataType transformation = selectTransformation(computation); + if (transformation != null) { + computation.getOutputMetadataValueBean().setTransformation(transformation.clone()); + } + } + + private TransformationMetadataType selectTransformation(ConsolidationMetadataComputation computation) { + for (ValueMetadataType nonNegativeValue : computation.getNonNegativeValues()) { + if (nonNegativeValue.getTransformation() != null) { + return nonNegativeValue.getTransformation(); + } + } + for (ValueMetadataType existingValue : computation.getExistingValues()) { + if (existingValue.getTransformation() != null) { + return existingValue.getTransformation(); + } + } + return null; + } +} diff --git a/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/TestValueMetadata.java b/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/TestValueMetadata.java index cd09a8c77ce..b0bd44eaab8 100644 --- a/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/TestValueMetadata.java +++ b/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/TestValueMetadata.java @@ -182,7 +182,7 @@ public void initSystem(Task initTask, OperationResult initResult) throws Excepti addObject(TEMPLATE_LOA_USER, initTask, initResult); addObject(USER_ALICE, initTask, initResult); - setGlobalTracingOverride(createModelLoggingTracingProfile()); +// setGlobalTracingOverride(createModelLoggingTracingProfile()); } @Override @@ -242,17 +242,18 @@ public void test020ParsingMetadata() throws Exception { .valueMetadata(UserType.F_NAME) .assertHasDefinition() .singleValue() - .containerSingle(ValueMetadataType.F_TRANSFORMATION) - .container(TransformationMetadataType.F_SOURCE) - .assertSize(1) - .singleValue() - .assertPropertyEquals(ValueSourceType.F_KIND, "http://midpoint.evolveum.com/data-provenance/source#resource") - .end() - .end() - .end() - .assertPropertyValuesEqual(LOA_PATH, 10) - .end() +// .containerSingle(ValueMetadataType.F_TRANSFORMATION) +// .container(TransformationMetadataType.F_SOURCE) +// .assertSize(1) +// .singleValue() +// .assertPropertyEquals(ValueSourceType.F_KIND, "http://midpoint.evolveum.com/data-provenance/source#resource") +// .end() +// .end() +// .end() +// .assertPropertyValuesEqual(LOA_PATH, 10) +// .end() .end() + .end() .valueMetadata(UserType.F_GIVEN_NAME) .assertHasDefinition() .singleValue() diff --git a/model/model-intest/src/test/resources/metadata/provenance-metadata-recording/template-provenance-metadata-recording.xml b/model/model-intest/src/test/resources/metadata/provenance-metadata-recording/template-provenance-metadata-recording.xml index 4bfab05fb6b..b90e9293930 100644 --- a/model/model-intest/src/test/resources/metadata/provenance-metadata-recording/template-provenance-metadata-recording.xml +++ b/model/model-intest/src/test/resources/metadata/provenance-metadata-recording/template-provenance-metadata-recording.xml @@ -36,19 +36,32 @@ - - fullName to alias - strong - - fullName - - - extension/alias - - matchingProvenance - - - + + extension/alias + + fullName to alias + strong + + fullName + + + + matchingProvenance + + + + + + + transformation + + full + + persistent + + + + @@ -118,6 +131,14 @@ + + transformation + + full + + transient + + extension/loa diff --git a/model/model-intest/src/test/resources/metadata/user-alice.xml b/model/model-intest/src/test/resources/metadata/user-alice.xml index 1edbf2dc730..c6f2706cc79 100644 --- a/model/model-intest/src/test/resources/metadata/user-alice.xml +++ b/model/model-intest/src/test/resources/metadata/user-alice.xml @@ -21,13 +21,16 @@ <_value>alice <_metadata> - - http://midpoint.evolveum.com/data-provenance/source#resource - Dummy Resource - - - name - + + + inbound name mapping + + + name + + alice + + 10 @@ -40,24 +43,24 @@ 5 - - http://midpoint.evolveum.com/data-provenance/source#userAction - willTurner - - + + + fullName mapping + + + title + pirate + + experimental <_value>manual <_metadata> - - - http://midpoint.evolveum.com/data-provenance/source#userAction - willTurner - - - + + 3 + @@ -84,13 +87,16 @@ <_value>Alice <_metadata> - - http://midpoint.evolveum.com/data-provenance/source#resource - Dummy Resource - - - givenName - + + + inbound givenName mapping + + + firstname + + Alice + + 7 @@ -101,13 +107,16 @@ <_value>Green <_metadata> - - http://midpoint.evolveum.com/data-provenance/source#resource - Dummy Resource - - - familyName - + + + inbound familyName mapping + + + lastname + + Green + + 7 @@ -118,13 +127,16 @@ <_value>Ph.D. <_metadata> - - http://midpoint.evolveum.com/data-provenance/source#resource - Dummy Resource - - - honorificSuffix - + + + inbound honorificSuffix mapping + + + aftertitle + + Ph.D. + + 5 @@ -135,32 +147,81 @@ <_value>Alice Green, Ph.D. <_metadata> - - http://midpoint.evolveum.com/data-provenance/source#resource - Dummy Resource - - - givenName - - - http://midpoint.evolveum.com/data-provenance/source#resource - Dummy Resource - - - familyName - - - http://midpoint.evolveum.com/data-provenance/source#resource - Dummy Resource - - - honorificSuffix - - - http://midpoint.evolveum.com/data-provenance/transformer#mapping - mapping-fullname - - + + + fullName mapping + + + + givenName + + <_value>Alice + <_metadata> + + + + inbound givenName mapping + + + firstname + + Alice + + + + + 7 + + + + + + familyName + + <_value>Green + <_metadata> + + + + inbound familyName mapping + + + lastname + + Green + + + + + 7 + + + + + + honorificSuffix + + <_value>Ph.D. + <_metadata> + + + + inbound honorificSuffix mapping + + + aftertitle + + Ph.D. + + + + + 5 + + + + + 5 @@ -171,14 +232,6 @@ <_value>Development <_metadata> - - - http://midpoint.evolveum.com/data-provenance/source#userAction - jack - - organizationalUnit - - 7 diff --git a/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/expression/evaluator/AbstractExpressionEvaluator.java b/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/expression/evaluator/AbstractExpressionEvaluator.java index 2fd3681542e..65612c3dd40 100644 --- a/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/expression/evaluator/AbstractExpressionEvaluator.java +++ b/repo/repo-common/src/main/java/com/evolveum/midpoint/repo/common/expression/evaluator/AbstractExpressionEvaluator.java @@ -192,7 +192,7 @@ public void addInternalOrigin(PrismValue value, ExpressionEvaluationContext cont public void applyValueMetadata(PrismValueDeltaSetTriple triple, ExpressionEvaluationContext context, OperationResult result) throws CommunicationException, ObjectNotFoundException, SchemaException, SecurityViolationException, ConfigurationException, ExpressionEvaluationException { - if (triple != null && context.getValueMetadataComputer() != null) { + if (triple != null) { for (V value : triple.getPlusSet()) { applyValueMetadata(value, context, result); } @@ -203,7 +203,9 @@ public void applyValueMetadata(PrismValueDeltaSetTriple triple, ExpressionEva // For values in minus set we must delete any existing metadata. The reason is that a mapping // computes own metadata for its outputs - so metadata from the input values in minus set are no longer // relevant on the output. - value.getValueMetadata().clear(); + if (value != null) { + value.getValueMetadata().clear(); + } } } } @@ -212,9 +214,15 @@ private void applyValueMetadata(PrismValue value, ExpressionEvaluationContext co throws CommunicationException, ObjectNotFoundException, SchemaException, SecurityViolationException, ConfigurationException, ExpressionEvaluationException { if (value != null) { - value.setValueMetadata( - context.getValueMetadataComputer() - .compute(Collections.singletonList(value), result)); + if (context.getValueMetadataComputer() != null) { + value.setValueMetadata( + context.getValueMetadataComputer() + .compute(Collections.singletonList(value), result)); + } else { + // This is to clear pre-existing metadata for asIs mappings that are not configured e.g. for provenance + // metadata propagation. + value.getValueMetadata().clear(); + } } } } diff --git a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/ObjectUpdater.java b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/ObjectUpdater.java index 3fa210caaef..84c4240088c 100644 --- a/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/ObjectUpdater.java +++ b/repo/repo-sql-impl/src/main/java/com/evolveum/midpoint/repo/sql/helpers/ObjectUpdater.java @@ -248,15 +248,14 @@ void updateFullObject(RObject object, PrismObject save .itemsToSkip(itemsToSkip) .options(SerializationOptions .createSerializeReferenceNamesForNullOids() - .skipIndexOnly(true)) + .skipIndexOnly(true) + .skipTransient(true)) .serialize(savedObject); byte[] fullObject = RUtil.getBytesFromSerializedForm(xml, getConfiguration().isUseZip()); object.setFullObject(fullObject); - if (LOGGER.isTraceEnabled()) { - LOGGER.trace("Updating full object xml column finished. Xml:\n{}", xml); - } + LOGGER.trace("Updating full object xml column finished. Xml:\n{}", xml); } protected SqlRepositoryConfiguration getConfiguration() { From 0b4307b70b44d300fe430411e0cfbb1f7b2ab112 Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Thu, 3 Sep 2020 17:30:47 +0200 Subject: [PATCH 06/12] Generalize basic.getMetadataExtensionValues method This is just a suggestion of how this method could look like. --- .../functions/BasicExpressionFunctions.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/functions/BasicExpressionFunctions.java b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/functions/BasicExpressionFunctions.java index 86bef80bc72..ebdbc402110 100644 --- a/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/functions/BasicExpressionFunctions.java +++ b/model/model-common/src/main/java/com/evolveum/midpoint/model/common/expression/functions/BasicExpressionFunctions.java @@ -39,6 +39,7 @@ import com.evolveum.midpoint.prism.path.ItemPath; import com.evolveum.midpoint.util.MiscUtil; import com.evolveum.midpoint.util.QNameUtil; +import com.evolveum.midpoint.util.annotation.Experimental; import com.evolveum.midpoint.xml.ns._public.common.common_3.*; import com.evolveum.prism.xml.ns._public.types_3.ItemPathType; import org.apache.commons.io.FileUtils; @@ -586,13 +587,18 @@ public Collection getAttributeStringValues(ShadowType shadow, javax.xml. return ShadowUtil.getAttributeValues(shadow, attributeQname, String.class); } + /** + * Generic method to extract all metadata values pointed-to by given item path (specified as segments). + * Note: does not support multivalued containers withing the path (e.g. collecting transformation/source/name, + * where transformation/source is a multivalued container). + */ + @Experimental @NotNull - public Collection getMetadataExtensionValues(PrismValue value, String itemLocalPart) { - checkColon(itemLocalPart); + public Collection getMetadataValues(PrismValue value, Object... pathSegments) { if (value == null) { return emptySet(); } else { - ItemPath itemPath = ItemPath.create(ValueMetadataType.F_EXTENSION, itemLocalPart); + ItemPath itemPath = ItemPath.create(pathSegments); return value.getValueMetadataAsContainer().valuesStream() .map(md -> md.findItem(itemPath)) .filter(Objects::nonNull) @@ -601,6 +607,16 @@ public Collection getMetadataExtensionValues(PrismValue value, String itemLoc } } + /** + * Simplified version of getMetadataValue aimed at fetching single-segment extension paths. + */ + @Experimental + @NotNull + public Collection getMetadataExtensionValues(PrismValue value, String itemLocalPart) { + checkColon(itemLocalPart); + return getMetadataValues(value, ValueMetadataType.F_EXTENSION, itemLocalPart); + } + public void setExtensionRealValues(PrismContainerValue containerValue, Map map) throws SchemaException { PrismContainer ext = containerValue.findOrCreateContainer(ObjectType.F_EXTENSION); Map qnameKeyedMap = new HashMap<>(); From e24a0df6e239366dcd4b44a2e561bbf3e1165373 Mon Sep 17 00:00:00 2001 From: lskublik Date: Thu, 3 Sep 2020 21:38:56 +0200 Subject: [PATCH 07/12] fix for TestInitialObjects --- .../com/evolveum/midpoint/web/TestInitialObjects.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gui/admin-gui/src/test/java/com/evolveum/midpoint/web/TestInitialObjects.java b/gui/admin-gui/src/test/java/com/evolveum/midpoint/web/TestInitialObjects.java index 066e2c23f47..0790670a22b 100644 --- a/gui/admin-gui/src/test/java/com/evolveum/midpoint/web/TestInitialObjects.java +++ b/gui/admin-gui/src/test/java/com/evolveum/midpoint/web/TestInitialObjects.java @@ -13,6 +13,7 @@ import com.evolveum.midpoint.prism.path.ItemPath; import com.evolveum.midpoint.schema.result.OperationResultStatus; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectCollectionType; import com.evolveum.midpoint.xml.ns._public.common.common_3.ReportType; import org.testng.annotations.Test; @@ -61,7 +62,7 @@ public void testInitialObjects() throws Exception { private void testInitialObject(ObjectValidator validator, StringBuilder errorsSb, File file) throws SchemaException, IOException { PrismObject object = getPrismContext().parseObject(file); ValidationResult validationResult = validator.validate(object); - if (validationResult.isEmpty() || isOnlyJasperWarning(validationResult)) { + if (validationResult.isEmpty() || isIgnoredWarning(validationResult)) { display("Checked "+object+": no warnings"); return; } @@ -76,9 +77,10 @@ private void testInitialObject(ObjectValidator validator, } } - private boolean isOnlyJasperWarning(ValidationResult validationResult) { + private boolean isIgnoredWarning(ValidationResult validationResult) { for (ValidationItem item : validationResult.getItems()){ - if (!item.getItemPath().equivalent(ItemPath.create(ReportType.F_JASPER)) + if ((!item.getItemPath().equivalent(ItemPath.create(ReportType.F_JASPER)) + && !item.getItemPath().equivalent(ItemPath.create(ObjectCollectionType.F_AUDIT_SEARCH))) || !item.getStatus().equals(OperationResultStatus.WARNING)){ return false; } From fbf491b6bcee3f943f6c529b5e9c93ffe4d93adc Mon Sep 17 00:00:00 2001 From: Richard Richter Date: Thu, 3 Sep 2020 23:01:26 +0200 Subject: [PATCH 08/12] AuditSearchTest.java: added test for LIKE with _, Querydsl covers it! _ in LIKE is "any character" (like . in regex) and needs to be escaped. If we use Querydsl operations like starts/endsWith() or contains(), the escaping is done automatically for us (not sure about like(), but that one has lower level semantics anyway). --- .../midpoint/repo/sql/AuditSearchTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/AuditSearchTest.java b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/AuditSearchTest.java index 42fbade17f3..85052b1c4a5 100644 --- a/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/AuditSearchTest.java +++ b/repo/repo-sql-impl-test/src/test/java/com/evolveum/midpoint/repo/sql/AuditSearchTest.java @@ -124,6 +124,7 @@ public void initSystem() throws Exception { record2.addDelta(createDeltaWithIgnoredPath(UserType.F_FAMILY_NAME)); record2.getCustomColumnProperty().put("foo", "foo-value-2"); record2.getCustomColumnProperty().put("bar", "bar-val"); + record2.setTaskOid("task_oid2"); auditService.audit(record2, NullTaskImpl.INSTANCE); AuditEventRecord record3 = new AuditEventRecord(); @@ -419,6 +420,7 @@ public void test141SearchByTimestampEqual() throws SchemaException { then("only audit events with the timestamp equal to are returned"); assertThat(result).hasSize(1); + // getParameter() is used to identify specific audit records (1, 2, 3) as mentioned in init assertThat(result.get(0).getParameter()).isEqualTo("2"); } @@ -680,6 +682,20 @@ public void test195SearchByTaskOid() throws SchemaException { assertThat(result).allMatch(r -> r.getTaskOID().equals("task-oid")); } + @Test + public void test196SearchByTaskOidLikeWithUnderscore() throws SchemaException { + when("searching audit filtered by task OID LIKE with underscore"); + SearchResultList result = searchObjects(prismContext + .queryFor(AuditEventRecordType.class) + // we double down here with ignore-casing matcher + .item(AuditEventRecordType.F_TASK_OID).startsWith("TASK_").matchingCaseIgnore() + .build()); + + then("only audit events with matching task OID are returned (underscore is escaped)"); + assertThat(result).hasSize(1); + assertThat(result.get(0).getParameter()).isEqualTo("2"); + } + /* * Our NOT means "complement" and must include NULL values for NOT(EQ...). * This is difference from SQL logic, where NOT x='value' does NOT return rows where x IS NULL. From 7dc98022c7c55f0b9ba794e96a5827df447bd854 Mon Sep 17 00:00:00 2001 From: kate Date: Fri, 4 Sep 2020 00:28:38 +0200 Subject: [PATCH 09/12] search filter title fix --- .../component/search/FilterSearchItem.java | 24 +++++++++++++++---- .../web/component/search/SearchItem.java | 4 +++- .../web/component/search/SearchItemPanel.java | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/FilterSearchItem.java b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/FilterSearchItem.java index b4cd3101272..a03f8dc0df5 100644 --- a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/FilterSearchItem.java +++ b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/FilterSearchItem.java @@ -6,19 +6,25 @@ */ package com.evolveum.midpoint.web.component.search; -import com.evolveum.midpoint.gui.api.util.WebComponentUtil; -import com.evolveum.midpoint.xml.ns._public.common.common_3.SearchItemType; - import org.apache.commons.lang.Validate; import org.apache.commons.lang.builder.ToStringBuilder; import org.jetbrains.annotations.NotNull; +import com.evolveum.midpoint.gui.api.page.PageBase; +import com.evolveum.midpoint.gui.api.util.WebComponentUtil; +import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.util.logging.LoggingUtils; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.midpoint.xml.ns._public.common.common_3.SearchItemType; + /** * @author honchar */ public class FilterSearchItem extends SearchItem { private static final long serialVersionUID = 1L; + private static final Trace LOGGER = TraceManager.getTrace(FilterSearchItem.class); public static final String F_APPLY_FILTER = "applyFilter"; @@ -42,8 +48,16 @@ public Type getType(){ } @Override - protected String getTitle(){ - return getPredefinedFilter().getFilter() != null ? getPredefinedFilter().getFilter().toString() : ""; + protected String getTitle(PageBase pageBase) { + if (getPredefinedFilter() == null || getPredefinedFilter().getFilter() == null) { + return null; + } + try { + return pageBase.getPrismContext().xmlSerializer().serializeRealValue(getPredefinedFilter().getFilter()); + } catch (SchemaException e) { + LoggingUtils.logUnexpectedException(LOGGER, "Cannot serialize filter", e); + } + return null; } public SearchItemType getPredefinedFilter() { diff --git a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchItem.java b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchItem.java index 5a53658fbab..31b0c9ce51b 100644 --- a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchItem.java +++ b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchItem.java @@ -7,6 +7,8 @@ package com.evolveum.midpoint.web.component.search; +import com.evolveum.midpoint.gui.api.page.PageBase; + import java.io.Serializable; /** @@ -33,7 +35,7 @@ public SearchItem(Search search) { public abstract Type getType(); - protected String getTitle(){ + protected String getTitle(PageBase pageBase){ return ""; } diff --git a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchItemPanel.java b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchItemPanel.java index e8b770fc224..68732664dd4 100644 --- a/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchItemPanel.java +++ b/gui/admin-gui/src/main/java/com/evolveum/midpoint/web/component/search/SearchItemPanel.java @@ -231,7 +231,7 @@ private IModel createTitleModel() { if (item == null){ return Model.of(); } - return Model.of(item.getTitle()); + return Model.of(item.getTitle(getPageBase())); } private void deletePerformed(AjaxRequestTarget target) { From 88bd8b2d44e1c6415c6e73ee1204ceacb466a2bd Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Fri, 4 Sep 2020 01:13:15 +0200 Subject: [PATCH 10/12] Improve trace processing 1) Add ninja option to analyze and transform traces 2) Improve visualization of assignment traces --- .../midpoint/schema/traces/OpNode.java | 6 +- .../midpoint/schema/traces/OpNodeFactory.java | 4 + .../schema/traces/OpNodeTreeBuilder.java | 24 ++- .../midpoint/schema/traces/OpType.java | 124 ++------------- .../schema/traces/TemplateExpander.java | 133 ++++++++++++++++ .../midpoint/schema/traces/TraceParser.java | 13 +- .../midpoint/schema/traces/TraceWriter.java | 51 +++++++ .../AssignmentEvaluationOpNode.java | 67 ++++++++ .../AssignmentSegmentEvaluationOpNode.java | 69 +++++++++ .../traces/operations/ResolutionUtil.java | 43 ++++++ .../midpoint/schema/util/AssignmentUtil.java | 143 ++++++++++++++++++ .../midpoint/schema/util/MappingUtil.java | 41 +++++ .../schema/util/OperationResultUtil.java | 50 ++++++ .../schema/util/PolicyRuleTypeUtil.java | 8 + .../resources/ctx-configuration-no-repo.xml | 55 +++++++ .../task/quartzimpl/tracing/TracerImpl.java | 28 ++-- .../ninja/action/trace/EditTraceAction.java | 141 +++++++++++++++++ .../ninja/action/trace/TraceStatistics.java | 126 +++++++++++++++ .../evolveum/midpoint/ninja/impl/Command.java | 5 +- .../midpoint/ninja/impl/NinjaContext.java | 15 +- .../ninja/opts/ConnectionOptions.java | 9 ++ .../midpoint/ninja/opts/EditTraceOptions.java | 68 +++++++++ .../src/main/resources/messages.properties | 10 +- 23 files changed, 1086 insertions(+), 147 deletions(-) create mode 100644 infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TemplateExpander.java create mode 100644 infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TraceWriter.java create mode 100644 infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/AssignmentEvaluationOpNode.java create mode 100644 infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/AssignmentSegmentEvaluationOpNode.java create mode 100644 infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/ResolutionUtil.java create mode 100644 infra/schema/src/main/java/com/evolveum/midpoint/schema/util/AssignmentUtil.java create mode 100644 infra/schema/src/main/java/com/evolveum/midpoint/schema/util/MappingUtil.java create mode 100644 infra/schema/src/main/java/com/evolveum/midpoint/schema/util/OperationResultUtil.java create mode 100644 repo/system-init/src/main/resources/ctx-configuration-no-repo.xml create mode 100644 tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/EditTraceAction.java create mode 100644 tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/TraceStatistics.java create mode 100644 tools/ninja/src/main/java/com/evolveum/midpoint/ninja/opts/EditTraceOptions.java diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNode.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNode.java index cc49cd8f8e6..3b914609d78 100644 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNode.java +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNode.java @@ -607,7 +607,7 @@ public int getMappingsCount() { } public Integer getAssignmentEvaluationsCount() { - return (int) getChildrenStream(2) + return (int) getChildrenStream(3) .filter(child -> child.getKind() == OperationKindType.ASSIGNMENT_EVALUATION) .count(); } @@ -639,4 +639,8 @@ public OpNodePresentation getPresentation() { public void setPresentation(OpNodePresentation presentation) { this.presentation = presentation; } + + public void resolveReferenceTargetNames(OpNodeTreeBuilder.NameResolver nameResolver) { + } + } diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNodeFactory.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNodeFactory.java index fc48a85de3e..46d02f95dad 100644 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNodeFactory.java +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNodeFactory.java @@ -45,6 +45,10 @@ public static OpNode createOpNode(PrismContext prismContext, OperationResultType case PROJECTOR_TEMPLATE_AFTER_ASSIGNMENTS: case PROJECTOR_COMPONENT_OTHER: return new ProjectorComponentOpNode(prismContext, result, info, parent, traceInfo); + case ASSIGNMENT_EVALUATION: + return new AssignmentEvaluationOpNode(prismContext, result, info, parent, traceInfo); + case ASSIGNMENT_SEGMENT_EVALUATION: + return new AssignmentSegmentEvaluationOpNode(prismContext, result, info, parent, traceInfo); } } return new OpNode(prismContext, result, info, parent, traceInfo); diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNodeTreeBuilder.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNodeTreeBuilder.java index ba4a8d0a3af..75256ef563e 100644 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNodeTreeBuilder.java +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpNodeTreeBuilder.java @@ -12,6 +12,8 @@ import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultType; import com.evolveum.midpoint.xml.ns._public.common.common_3.TracingOutputType; +import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; + import org.jetbrains.annotations.NotNull; import java.util.ArrayList; @@ -23,25 +25,35 @@ public class OpNodeTreeBuilder { @NotNull private final PrismContext prismContext; - private IdentityHashMap infoMap = new IdentityHashMap<>(); + private final IdentityHashMap infoMap = new IdentityHashMap<>(); - public OpNodeTreeBuilder(PrismContext prismContext) { + public OpNodeTreeBuilder(@NotNull PrismContext prismContext) { this.prismContext = prismContext; } public List build(TracingOutputType tracingOutput) { + return build(tracingOutput, null); + } + + public List build(TracingOutputType tracingOutput, NameResolver nameResolver) { List rv = new ArrayList<>(); - addNode(null, rv, tracingOutput.getResult(), new TraceInfo(tracingOutput)); + addNode(null, rv, tracingOutput.getResult(), new TraceInfo(tracingOutput), nameResolver); return rv; - } - private void addNode(OpNode parent, List rv, OperationResultType result, TraceInfo traceInfo) { + private void addNode(OpNode parent, List rv, OperationResultType result, TraceInfo traceInfo, NameResolver nameResolver) { OpResultInfo info = OpResultInfo.create(result, infoMap); OpNode newNode = OpNodeFactory.createOpNode(prismContext, result, info, parent, traceInfo); + if (nameResolver != null) { + newNode.resolveReferenceTargetNames(nameResolver); + } rv.add(newNode); for (OperationResultType child : result.getPartialResults()) { - addNode(newNode, newNode.getChildren(), child, traceInfo); + addNode(newNode, newNode.getChildren(), child, traceInfo, nameResolver); } } + + public interface NameResolver { + PolyStringType getName(String oid); + } } diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpType.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpType.java index ae6301da554..967acdbe747 100644 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpType.java +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/OpType.java @@ -59,9 +59,17 @@ public enum OpType { "com.evolveum.midpoint.model.impl.lens.projector.Projector.assignments", "Assignments (${m:getAssignmentEvaluationsCount})"), - ASSIGNMENT_EVALUATION(OperationKindType.ASSIGNMENT_EVALUATION, "Assignment evaluation", + ASSIGNMENT_EVALUATION_OUTER(OperationKindType.OTHER, "Assignment evaluation (outer)", "com.evolveum.midpoint.model.impl.lens.projector.focus.AssignmentTripleEvaluator.evaluateAssignment", - "Assignment evaluation (→ ${c:assignmentTargetName})"), + "Assignment evaluation: → ${c:assignmentTargetName}"), + + ASSIGNMENT_EVALUATION(OperationKindType.ASSIGNMENT_EVALUATION, "Assignment evaluation", + "com.evolveum.midpoint.model.impl.lens.AssignmentEvaluator.evaluate", + "Assignment evaluation: ${m:getAssignmentInfo}"), + + ASSIGNMENT_SEGMENT_EVALUATION(OperationKindType.OTHER, "Assignment segment evaluation", + "com.evolveum.midpoint.model.impl.lens.AssignmentEvaluator.evaluateFromSegment", + "${m:getSegmentLabel}"), PROJECTION_ACTIVATION(OperationKindType.OTHER,"Projection activation", "com.evolveum.midpoint.model.impl.lens.projector.ActivationProcessor.projectionActivation", @@ -192,7 +200,8 @@ private static boolean isLoadedFromRepository(OperationResultType result) { public String getFormattedName(OpNode node) { if (nameTemplate != null) { - return expandTemplate(node); + return new TemplateExpander() + .expandTemplate(node, nameTemplate); } OperationResultType opResult = node.getResult(); @@ -214,12 +223,6 @@ public String getFormattedName(OpNode node) { String srcName = getContext(opResult, "segmentSourceName"); String tgtName = getContext(opResult, "segmentTargetName"); return "Segment: " + (srcName != null ? srcName + " " : "") + " → " + (tgtName != null ? " " + tgtName : ""); - } else if ("com.evolveum.midpoint.model.impl.lens.AssignmentEvaluator.evaluate".equals(operation)) { - String tgtName = getContext(opResult, "assignmentTargetName"); - return "AssignmentEvaluator.evaluate" + (tgtName != null ? " (→ " + tgtName + ")" : ""); - } else if ("com.evolveum.midpoint.model.impl.lens.projector.focus.AssignmentTripleEvaluator.evaluateAssignment".equals(operation)) { - String tgtName = getContext(opResult, "assignmentTargetName"); - return "AssignmentTripleEvaluator.evaluateAssignment" + (tgtName != null ? " (→ " + tgtName + ")" : ""); } else if ("com.evolveum.midpoint.model.impl.lens.projector.policy.PolicyRuleProcessor.evaluateRule".equals(operation)) { String triggeredString = getReturn(opResult, "triggered"); int enabledActions = TraceUtil.getReturnsAsStringList(opResult, "enabledActions").size(); @@ -261,109 +264,6 @@ public String getFormattedName(OpNode node) { return opResult.getOperation() + (qualifiers.isEmpty() ? "" : " (" + qualifiers + ")"); } - private String expandTemplate(OpNode node) { - return new StringSubstitutor(createResolver(node), "${", "}", '\\') - .replace(nameTemplate); - } - - private StringLookup createResolver(OpNode node) { - return spec -> { - String prefix; - String suffix; - String key; - - String[] parts = spec.split(":"); - if (parts.length == 1) { - prefix = ""; - key = spec; - suffix = ""; - } else if (parts.length == 2) { - prefix = parts[0]; - key = parts[1]; - suffix = ""; - } else if (parts.length == 3) { - prefix = parts[0]; - key = parts[1]; - suffix = parts[2]; - } else { - return "???"; - } - - List values = new ArrayList<>(); - if (prefix.isEmpty() || prefix.equals("p")) { - collectMatchingParams(values, key, node.getResult().getParams()); - } - if (prefix.isEmpty() || prefix.equals("c")) { - collectMatchingParams(values, key, node.getResult().getContext()); - } - if (prefix.isEmpty() || prefix.equals("r")) { - collectMatchingParams(values, key, node.getResult().getReturns()); - } - if (prefix.isEmpty() || prefix.equals("t")) { - collectMatchingValues(values, key, node.getResult().getTrace()); - } - if (prefix.equals("m")) { - collectFromMethod(values, key, node); - } - return String.join(", ", postprocess(values, suffix)); - }; - } - - private void collectFromMethod(List values, String key, OpNode node) { - try { - Object rv = MethodUtils.invokeExactMethod(node, key); - if (rv instanceof Collection) { - for (Object o : (Collection) rv) { - values.add(String.valueOf(o)); - } - } else if (rv != null) { - values.add(String.valueOf(rv)); - } - } catch (Throwable t) { - values.add("??? " + t.getMessage()); - } - } - - private void collectMatchingValues(List values, String path, List traces) { - UniformItemPath itemPath = ItemPathParserTemp.parseFromString(path); // FIXME (hack) - for (TraceType trace : traces) { - PrismProperty property = trace.asPrismContainerValue().findProperty(itemPath); - if (property != null) { - for (Object realValue : property.getRealValues()) { - values.add(String.valueOf(realValue)); - } - } - } - } - - private void collectMatchingParams(List values, String key, ParamsType params) { - int colon = key.indexOf(':'); - String processing; - String realKey; - if (colon >= 0) { - realKey = key.substring(0, colon); - processing = key.substring(colon + 1); - } else { - realKey = key; - processing = ""; - } - List rawStringValues = asStringList(selectByKey(params, realKey)); - values.addAll(postprocess(rawStringValues, processing)); - } - - private List postprocess(List raw, String processing) { - return raw.stream() - .map(s -> postprocess(s, processing)) - .collect(Collectors.toList()); - } - - private String postprocess(String string, String processing) { - if (processing.contains("L")) { - string = string.toLowerCase(); - } - return string; - } - private String getRepoCacheOpDescription(OpNode node, OperationResultType opResult, String last, String commaQualifiers) { String postfix = ""; if ("getObject".equals(last)) { diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TemplateExpander.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TemplateExpander.java new file mode 100644 index 00000000000..5fb61f0b0c1 --- /dev/null +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TemplateExpander.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.schema.traces; + +import com.evolveum.midpoint.prism.PrismProperty; +import com.evolveum.midpoint.prism.impl.marshaller.ItemPathParserTemp; +import com.evolveum.midpoint.prism.path.UniformItemPath; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ParamsType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.TraceType; + +import org.apache.commons.lang3.reflect.MethodUtils; +import org.apache.commons.text.StringSubstitutor; +import org.apache.commons.text.lookup.StringLookup; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static com.evolveum.midpoint.schema.traces.TraceUtil.asStringList; +import static com.evolveum.midpoint.schema.traces.TraceUtil.selectByKey; + +public class TemplateExpander { + + public String expandTemplate(OpNode node, String nameTemplate) { + return new StringSubstitutor(createResolver(node), "${", "}", '\\') + .replace(nameTemplate); + } + + private StringLookup createResolver(OpNode node) { + return spec -> { + String prefix; + String suffix; + String key; + + String[] parts = spec.split(":"); + if (parts.length == 1) { + prefix = ""; + key = spec; + suffix = ""; + } else if (parts.length == 2) { + prefix = parts[0]; + key = parts[1]; + suffix = ""; + } else if (parts.length == 3) { + prefix = parts[0]; + key = parts[1]; + suffix = parts[2]; + } else { + return "???"; + } + + List values = new ArrayList<>(); + if (prefix.isEmpty() || prefix.equals("p")) { + collectMatchingParams(values, key, node.getResult().getParams()); + } + if (prefix.isEmpty() || prefix.equals("c")) { + collectMatchingParams(values, key, node.getResult().getContext()); + } + if (prefix.isEmpty() || prefix.equals("r")) { + collectMatchingParams(values, key, node.getResult().getReturns()); + } + if (prefix.isEmpty() || prefix.equals("t")) { + collectMatchingValues(values, key, node.getResult().getTrace()); + } + if (prefix.equals("m")) { + collectFromMethod(values, key, node); + } + return String.join(", ", postprocess(values, suffix)); + }; + } + + private void collectFromMethod(List values, String key, OpNode node) { + try { + Object rv = MethodUtils.invokeExactMethod(node, key); + if (rv instanceof Collection) { + for (Object o : (Collection) rv) { + values.add(String.valueOf(o)); + } + } else if (rv != null) { + values.add(String.valueOf(rv)); + } + } catch (Throwable t) { + values.add("??? " + t.getMessage()); + } + } + + private void collectMatchingValues(List values, String path, List traces) { + UniformItemPath itemPath = ItemPathParserTemp.parseFromString(path); // FIXME (hack) + for (TraceType trace : traces) { + PrismProperty property = trace.asPrismContainerValue().findProperty(itemPath); + if (property != null) { + for (Object realValue : property.getRealValues()) { + values.add(String.valueOf(realValue)); + } + } + } + } + + private void collectMatchingParams(List values, String key, ParamsType params) { + int colon = key.indexOf(':'); + String processing; + String realKey; + if (colon >= 0) { + realKey = key.substring(0, colon); + processing = key.substring(colon + 1); + } else { + realKey = key; + processing = ""; + } + List rawStringValues = asStringList(selectByKey(params, realKey)); + values.addAll(postprocess(rawStringValues, processing)); + } + + private List postprocess(List raw, String processing) { + return raw.stream() + .map(s -> postprocess(s, processing)) + .collect(Collectors.toList()); + } + + private String postprocess(String string, String processing) { + if (processing.contains("L")) { + string = string.toLowerCase(); + } + return string; + } + +} diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TraceParser.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TraceParser.java index 7176ec156f9..7351cdb22fd 100644 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TraceParser.java +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TraceParser.java @@ -32,19 +32,26 @@ public class TraceParser { @NotNull private final PrismContext prismContext; - @SuppressWarnings("WeakerAccess") // used externally public TraceParser(@NotNull PrismContext prismContext) { this.prismContext = prismContext; } public TracingOutputType parse(File file) throws IOException, SchemaException { + return parse(file, false); + } + + public TracingOutputType parse(File file, boolean raw) throws IOException, SchemaException { boolean isZip = file.getName().toLowerCase().endsWith(".zip"); - return parse(new FileInputStream(file), isZip, file.getPath()); + return parse(new FileInputStream(file), isZip, raw, file.getPath()); } public TracingOutputType parse(InputStream inputStream, boolean isZip, String description) throws SchemaException, IOException { + return parse(inputStream, isZip, false, description); + } + + public TracingOutputType parse(InputStream inputStream, boolean isZip, boolean raw, String description) throws SchemaException, IOException { TracingOutputType wholeTracingOutput = getObject(inputStream, isZip, description); - if (wholeTracingOutput != null) { + if (!raw && wholeTracingOutput != null) { new DictionaryExpander(wholeTracingOutput).expand(); new OperationCategorizer(wholeTracingOutput).categorize(); } diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TraceWriter.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TraceWriter.java new file mode 100644 index 00000000000..7789ffdd10f --- /dev/null +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/TraceWriter.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.schema.traces; + +import java.io.*; +import java.nio.charset.StandardCharsets; + +import com.evolveum.midpoint.prism.SerializationOptions; +import com.evolveum.midpoint.util.MiscUtil; + +import org.jetbrains.annotations.NotNull; + +import com.evolveum.midpoint.prism.PrismContext; +import com.evolveum.midpoint.util.annotation.Experimental; +import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.xml.ns._public.common.common_3.TracingOutputType; + +@Experimental +public class TraceWriter { + + private static final String ZIP_ENTRY_NAME = "trace.xml"; + + @NotNull private final PrismContext prismContext; + + public TraceWriter(@NotNull PrismContext prismContext) { + this.prismContext = prismContext; + } + + @NotNull + public String writeTrace(TracingOutputType tracingOutput, File file, boolean zip) throws SchemaException, IOException { + String xml = prismContext.xmlSerializer() + .options( + SerializationOptions + .createSerializeReferenceNames() + .escapeInvalidCharacters(true)) + .serializeRealValue(tracingOutput); + if (zip) { + MiscUtil.writeZipFile(file, ZIP_ENTRY_NAME, xml, StandardCharsets.UTF_8); + } else { + try (PrintWriter pw = new PrintWriter(new FileWriter(file))) { + pw.write(xml); + } + } + return xml; + } +} diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/AssignmentEvaluationOpNode.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/AssignmentEvaluationOpNode.java new file mode 100644 index 00000000000..c2ab79bd4ef --- /dev/null +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/AssignmentEvaluationOpNode.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.schema.traces.operations; + +import static com.evolveum.midpoint.schema.traces.operations.ResolutionUtil.resolveAssignmentReferenceNames; + +import com.evolveum.midpoint.prism.PrismContext; +import com.evolveum.midpoint.schema.traces.*; +import com.evolveum.midpoint.schema.util.AssignmentUtil; +import com.evolveum.midpoint.xml.ns._public.common.common_3.AssignmentEvaluationTraceType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.AssignmentType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultType; + +public class AssignmentEvaluationOpNode extends OpNode { + + private final AssignmentEvaluationTraceType trace; + private final AssignmentType assignment; + + public AssignmentEvaluationOpNode(PrismContext prismContext, + OperationResultType result, + OpResultInfo info, OpNode parent, + TraceInfo traceInfo) { + super(prismContext, result, info, parent, traceInfo); + trace = getTrace(AssignmentEvaluationTraceType.class); + assignment = getAssignmentAny(); + } + + private AssignmentType getAssignmentAny() { + if (trace != null) { + if (trace.getAssignmentNew() != null) { + return trace.getAssignmentNew(); + } else { + return trace.getAssignmentOld(); + } + } else { + return null; + } + } + + public AssignmentEvaluationTraceType getTrace() { + return trace; + } + + public AssignmentType getAssignment() { + return assignment; + } + + public String getAssignmentInfo() { + if (assignment != null) { + return "→ " + AssignmentUtil.getAssignmentInfo(assignment); + } else { + return new TemplateExpander() + .expandTemplate(this, "→ ${c:assignmentTargetName}"); + } + } + + @Override + public void resolveReferenceTargetNames(OpNodeTreeBuilder.NameResolver nameResolver) { + super.resolveReferenceTargetNames(nameResolver); + resolveAssignmentReferenceNames(assignment, nameResolver); + } +} diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/AssignmentSegmentEvaluationOpNode.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/AssignmentSegmentEvaluationOpNode.java new file mode 100644 index 00000000000..afa1f828e16 --- /dev/null +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/AssignmentSegmentEvaluationOpNode.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.schema.traces.operations; + +import com.evolveum.midpoint.prism.PrismContext; +import com.evolveum.midpoint.schema.traces.*; +import com.evolveum.midpoint.schema.util.AssignmentUtil; +import com.evolveum.midpoint.xml.ns._public.common.common_3.*; +import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; + +import static com.evolveum.midpoint.schema.traces.operations.ResolutionUtil.resolveAssignmentReferenceNames; +import static com.evolveum.midpoint.schema.traces.operations.ResolutionUtil.resolveReferenceName; + +public class AssignmentSegmentEvaluationOpNode extends OpNode { + + private final AssignmentSegmentEvaluationTraceType trace; + private final AssignmentPathSegmentType segment; + + public AssignmentSegmentEvaluationOpNode(PrismContext prismContext, + OperationResultType result, + OpResultInfo info, OpNode parent, + TraceInfo traceInfo) { + super(prismContext, result, info, parent, traceInfo); + trace = getTrace(AssignmentSegmentEvaluationTraceType.class); + segment = trace != null ? trace.getSegment() : null; + + fixSegmentTargetName(); + } + + private void fixSegmentTargetName() { + if (segment != null && segment.getTargetRef() != null && segment.getTargetRef().getTargetName() == null) { + String targetName = TraceUtil.getContext(result, "segmentTargetName"); + if (targetName != null) { + segment.getTargetRef().setTargetName(PolyStringType.fromOrig(targetName)); + } + } + } + + public AssignmentSegmentEvaluationTraceType getTrace() { + return trace; + } + + public AssignmentPathSegmentType getSegment() { + return segment; + } + + public String getSegmentLabel() { + if (segment != null) { + return AssignmentUtil.getSegmentInfo(segment); + } else { + return new TemplateExpander() + .expandTemplate(this, "Segment: ${c:segmentSourceName} → ${c:segmentTargetName}"); + } + } + + @Override + public void resolveReferenceTargetNames(OpNodeTreeBuilder.NameResolver nameResolver) { + super.resolveReferenceTargetNames(nameResolver); + if (segment != null) { + resolveReferenceName(segment.getTargetRef(), nameResolver); + resolveAssignmentReferenceNames(segment.getAssignment(), nameResolver); + } + } +} diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/ResolutionUtil.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/ResolutionUtil.java new file mode 100644 index 00000000000..b5b6c979e52 --- /dev/null +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/traces/operations/ResolutionUtil.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.schema.traces.operations; + +import com.evolveum.midpoint.schema.traces.OpNodeTreeBuilder; +import com.evolveum.midpoint.util.annotation.Experimental; +import com.evolveum.midpoint.xml.ns._public.common.common_3.AssignmentType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectReferenceType; +import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; + +@Experimental +public class ResolutionUtil { + + private static final String COULD_NOT_RESOLVE = ResolutionUtil.class.getName() + ".couldNotResolve"; + + public static void resolveReferenceName(ObjectReferenceType reference, OpNodeTreeBuilder.NameResolver nameResolver) { + if (nameResolver != null && reference != null && reference.getOid() != null && reference.getTargetName() == null) { + if (reference.asReferenceValue().getUserData(COULD_NOT_RESOLVE) != null) { + return; + } + PolyStringType targetName = nameResolver.getName(reference.getOid()); + if (targetName != null) { + reference.setTargetName(targetName); + } else { + reference.asReferenceValue().setUserData(COULD_NOT_RESOLVE, true); + } + } + } + + public static void resolveAssignmentReferenceNames(AssignmentType assignment, OpNodeTreeBuilder.NameResolver nameResolver) { + if (nameResolver != null && assignment != null) { + resolveReferenceName(assignment.getTargetRef(), nameResolver); + if (assignment.getConstruction() != null) { + resolveReferenceName(assignment.getConstruction().getResourceRef(), nameResolver); + } + } + } +} diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/AssignmentUtil.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/AssignmentUtil.java new file mode 100644 index 00000000000..8da5e81564f --- /dev/null +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/AssignmentUtil.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.schema.util; + +import com.evolveum.midpoint.schema.constants.SchemaConstants; +import com.evolveum.midpoint.util.QNameUtil; +import com.evolveum.midpoint.util.annotation.Experimental; +import com.evolveum.midpoint.xml.ns._public.common.common_3.*; + +import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +public class AssignmentUtil { + + @Experimental + public static String getAssignmentInfo(AssignmentType assignment) { + if (assignment == null) { + return null; + } + List infos = new ArrayList<>(); + CollectionUtils.addIgnoreNull(infos, getReferenceInfo(assignment.getTargetRef())); + CollectionUtils.addIgnoreNull(infos, getConstructionInfo(assignment.getConstruction())); + infos.addAll(getMappingsInfo(assignment.getFocusMappings())); + CollectionUtils.addIgnoreNull(infos, PolicyRuleTypeUtil.toShortString(assignment.getPolicyRule())); + + return String.join("; ", infos); + } + + private static List getMappingsInfo(MappingsType focusMappings) { + if (focusMappings != null) { + return focusMappings.getMapping().stream() + .map(mapping -> "m:" + MappingUtil.getShortInfo(mapping)) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private static String getConstructionInfo(ConstructionType construction) { + if (construction != null) { + StringBuilder sb = new StringBuilder(); + sb.append("c:") + .append(getReferenceInfo(construction.getResourceRef())); + if (construction.getKind() != null) { + sb.append(", ").append(construction.getKind()); + } + if (construction.getIntent() != null) { + sb.append(", ").append(construction.getIntent()); + } + return sb.toString(); + } else { + return null; + } + } + + private static String getReferenceInfo(ObjectReferenceType targetRef) { + if (targetRef != null) { + StringBuilder sb = new StringBuilder(); + PolyStringType targetName = targetRef.getTargetName(); + if (targetName != null) { + sb.append(targetName.getOrig()); + } else if (targetRef.getOid() != null) { + sb.append(targetRef.getOid()); + } else if (targetRef.getFilter() != null) { + sb.append(targetRef.getFilter()); // todo + } else { + sb.append("(empty)"); + } + + if (targetRef.getRelation() != null && !QNameUtil.match(targetRef.getRelation(), SchemaConstants.ORG_DEFAULT)) { + sb.append(" [").append(targetRef.getRelation().getLocalPart()).append("]"); + } + return sb.toString(); + } else { + return null; + } + } + + @Experimental + public static String getSegmentInfo(AssignmentPathSegmentType segment) { + if (segment == null) { + return null; + } + if (segment.getAssignment() == null) { + return "no assignment"; + } + + StringBuilder sb = new StringBuilder(); + if (Boolean.FALSE.equals(segment.isIsAssignment())) { + sb.append("Inducement: "); + } else { + sb.append("Assignment: "); + } + sb.append(getReferenceInfo(segment.getSourceRef())); + sb.append(" → "); + sb.append(getAssignmentInfo(segment.getAssignment())); + + if (segment.getAssignment().getOrder() != null || !segment.getAssignment().getOrderConstraint().isEmpty()) { + sb.append(" (").append(getOrderInfo(segment.getAssignment())).append(")"); + } + + return sb.toString(); + } + + private static String getOrderInfo(AssignmentType assignment) { + List constraints = new ArrayList<>(); + if (assignment.getOrder() != null) { + constraints.add("order " + assignment.getOrder()); + } + for (OrderConstraintsType orderConstraint : assignment.getOrderConstraint()) { + constraints.add(getOrderConstraintInfo(orderConstraint)); + } + return String.join("; ", constraints); + } + + private static String getOrderConstraintInfo(OrderConstraintsType constraint) { + StringBuilder sb = new StringBuilder(); + if (constraint.getRelation() != null) { + sb.append(constraint.getRelation().getLocalPart()).append("="); + } + if (constraint.getOrder() != null) { + sb.append(constraint.getOrder()); + } + if (constraint.getOrderMin() != null || constraint.getOrderMax() != null) { + sb.append("<").append(constraint.getOrderMin()).append(",").append(constraint.getOrderMax()).append(">"); + } + if (constraint.getResetOrder() != null) { + sb.append("!").append(constraint.getResetOrder()); + } + return sb.toString(); + } +} diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/MappingUtil.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/MappingUtil.java new file mode 100644 index 00000000000..5f9d1c24756 --- /dev/null +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/MappingUtil.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.schema.util; + +import com.evolveum.midpoint.util.annotation.Experimental; +import com.evolveum.midpoint.xml.ns._public.common.common_3.AbstractMappingType; + +import java.util.Objects; +import java.util.stream.Collectors; + +@Experimental +public class MappingUtil { + + public static String getShortInfo(AbstractMappingType mapping) { + if (mapping != null) { + if (mapping.getName() != null) { + return mapping.getName(); + } else if (mapping.getTarget() != null && mapping.getTarget().getPath() != null) { + return "→ " + mapping.getTarget().getPath(); + } else if (!mapping.getSource().isEmpty()) { + return getSourcesInfo(mapping); + } else { + return "mapping"; + } + } else { + return null; + } + } + + private static String getSourcesInfo(AbstractMappingType mapping) { + return mapping.getSource().stream() + .filter(Objects::nonNull) + .map(source -> String.valueOf(source.getPath())) + .collect(Collectors.joining(", ")); + } +} diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/OperationResultUtil.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/OperationResultUtil.java new file mode 100644 index 00000000000..50b69d48edc --- /dev/null +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/OperationResultUtil.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.schema.util; + +import com.evolveum.midpoint.prism.util.CloneUtil; +import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultType; + +public class OperationResultUtil { + + public static OperationResultType shallowClone(OperationResultType result, boolean subresults, boolean traces, boolean log) { + OperationResultType clone = new OperationResultType(); + clone.setOperation(result.getOperation()); + clone.getQualifier().addAll(result.getQualifier()); + clone.setOperationKind(result.getOperationKind()); + clone.setStatus(result.getStatus()); + clone.setImportance(result.getImportance()); + clone.setMinor(result.isMinor()); + clone.setAsynchronousOperationReference(result.getAsynchronousOperationReference()); + clone.setStart(CloneUtil.clone(result.getStart())); + clone.setEnd(CloneUtil.clone(result.getEnd())); + clone.setMicroseconds(result.getMicroseconds()); + clone.setInvocationId(result.getInvocationId()); + clone.setTraced(result.isTraced()); + if (traces) { + clone.getTrace().addAll(result.getTrace()); + } + clone.setCount(result.getCount()); + clone.setHiddenRecordsCount(result.getHiddenRecordsCount()); + clone.setParams(result.getParams()); + clone.setContext(result.getContext()); + clone.setReturns(result.getReturns()); + clone.setToken(result.getToken()); + clone.setMessageCode(result.getMessageCode()); + clone.setMessage(result.getMessage()); + clone.setUserFriendlyMessage(result.getUserFriendlyMessage()); + clone.setDetails(result.getDetails()); + if (log) { + clone.getLog().addAll(result.getLog()); + } + if (subresults) { + clone.getPartialResults().addAll(result.getPartialResults()); + } + return clone; + } +} diff --git a/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/PolicyRuleTypeUtil.java b/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/PolicyRuleTypeUtil.java index 7c014d82161..829d4679689 100644 --- a/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/PolicyRuleTypeUtil.java +++ b/infra/schema/src/main/java/com/evolveum/midpoint/schema/util/PolicyRuleTypeUtil.java @@ -77,6 +77,14 @@ public class PolicyRuleTypeUtil { CONSTRAINT_NAMES.put(PolicyConstraintsType.F_ALWAYS_TRUE.getLocalPart(), SYMBOL_ALWAYS_TRUE); } + public static String toShortString(PolicyRuleType rule) { + if (rule != null) { + return toShortString(rule.getPolicyConstraints()) + "→" + toShortString(rule.getPolicyActions()); + } else { + return null; + } + } + public static String toShortString(PolicyConstraintsType constraints) { return toShortString(constraints, JOIN_AND); } diff --git a/repo/system-init/src/main/resources/ctx-configuration-no-repo.xml b/repo/system-init/src/main/resources/ctx-configuration-no-repo.xml new file mode 100644 index 00000000000..667f7229e23 --- /dev/null +++ b/repo/system-init/src/main/resources/ctx-configuration-no-repo.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/repo/task-quartz-impl/src/main/java/com/evolveum/midpoint/task/quartzimpl/tracing/TracerImpl.java b/repo/task-quartz-impl/src/main/java/com/evolveum/midpoint/task/quartzimpl/tracing/TracerImpl.java index 1a2667abffa..2640b2bec20 100644 --- a/repo/task-quartz-impl/src/main/java/com/evolveum/midpoint/task/quartzimpl/tracing/TracerImpl.java +++ b/repo/task-quartz-impl/src/main/java/com/evolveum/midpoint/task/quartzimpl/tracing/TracerImpl.java @@ -8,10 +8,7 @@ package com.evolveum.midpoint.task.quartzimpl.tracing; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.Objects; import java.util.*; @@ -20,6 +17,8 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import com.evolveum.midpoint.schema.traces.TraceWriter; + import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.commons.text.StringSubstitutor; @@ -31,7 +30,6 @@ import com.evolveum.midpoint.common.configuration.api.MidpointConfiguration; import com.evolveum.midpoint.prism.PrismContext; -import com.evolveum.midpoint.prism.SerializationOptions; import com.evolveum.midpoint.repo.api.RepositoryService; import com.evolveum.midpoint.repo.api.SystemConfigurationChangeDispatcher; import com.evolveum.midpoint.repo.api.SystemConfigurationChangeListener; @@ -41,7 +39,6 @@ import com.evolveum.midpoint.task.api.Task; import com.evolveum.midpoint.task.api.TaskManager; import com.evolveum.midpoint.task.api.Tracer; -import com.evolveum.midpoint.util.MiscUtil; import com.evolveum.midpoint.util.exception.ObjectAlreadyExistsException; import com.evolveum.midpoint.util.exception.SchemaException; import com.evolveum.midpoint.util.exception.SystemException; @@ -66,7 +63,7 @@ public class TracerImpl implements Tracer, SystemConfigurationChangeListener { private static final String MACRO_RANDOM = "random"; // To be used during tests to check for MID-5851 - @SuppressWarnings("WeakerAccess") @VisibleForTesting + @VisibleForTesting public static boolean checkHashCodeEqualsRelation = false; @Autowired private PrismContext prismContext; @@ -83,7 +80,6 @@ public class TracerImpl implements Tracer, SystemConfigurationChangeListener { private static final String OP_STORE_TRACE = TracerImpl.class.getName() + ".storeTrace"; private static final String TRACE_DIR_NAME = "trace"; - private static final String ZIP_ENTRY_NAME = "trace.xml"; private static final String DEFAULT_FILE_NAME_PATTERN = "trace-%{timestamp}"; @@ -122,23 +118,17 @@ public void storeTrace(Task task, OperationResult result, @Nullable OperationRes try { long start = System.currentTimeMillis(); TracingOutputType tracingOutput = tracingOutputCreator.createTracingOutput(task, result, tracingProfile); - String xml = prismContext.xmlSerializer() - .options( - SerializationOptions - .createSerializeReferenceNames() - .escapeInvalidCharacters(true)) - .serializeRealValue(tracingOutput); + String xml = new TraceWriter(prismContext) + .writeTrace(tracingOutput, file, zip); + if (zip) { - MiscUtil.writeZipFile(file, ZIP_ENTRY_NAME, xml, StandardCharsets.UTF_8); LOGGER.info("Trace was written to {} ({} chars uncompressed) in {} milliseconds", file, xml.length(), System.currentTimeMillis() - start); } else { - try (PrintWriter pw = new PrintWriter(new FileWriter(file))) { - pw.write(xml); - LOGGER.info("Trace was written to {} ({} chars) in {} milliseconds", file, xml.length(), - System.currentTimeMillis() - start); - } + LOGGER.info("Trace was written to {} ({} chars) in {} milliseconds", file, xml.length(), + System.currentTimeMillis() - start); } + if (!Boolean.FALSE.equals(tracingProfile.isCreateRepoObject())) { ReportDataType reportDataObject = new ReportDataType(prismContext) .name(createObjectName(tracingProfile, templateParameters)) diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/EditTraceAction.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/EditTraceAction.java new file mode 100644 index 00000000000..a11143eeabc --- /dev/null +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/EditTraceAction.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ +package com.evolveum.midpoint.ninja.action.trace; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.regex.Pattern; + +import com.evolveum.midpoint.ninja.action.RepositoryAction; +import com.evolveum.midpoint.ninja.opts.EditTraceOptions; + +import com.evolveum.midpoint.schema.traces.TraceParser; + +import com.evolveum.midpoint.schema.traces.TraceWriter; +import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.TracingOutputType; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; + +/** + * TODO + */ +public class EditTraceAction extends RepositoryAction { + + private static final String DEFAULT_OUTPUT = "output.zip"; + + private int killed; + + @Override + public void execute() throws Exception { + TracingOutputType trace = parseInput(); + if (options.isPrintStat() || options.isPrintStatExtra()) { + printStatistics(trace); + } + + if (CollectionUtils.isNotEmpty(options.getKeep()) || CollectionUtils.isNotEmpty(options.getKill())) { + applyKeep(trace); + applyKill(trace); + writeTrace(trace); + } + } + + private TracingOutputType parseInput() throws IOException, SchemaException { + String inputFile = options.getInput(); + log.info("Starting parsing input file: {}", inputFile); + + long start = System.currentTimeMillis(); + TraceParser parser = new TraceParser(context.getPrismContext()); + TracingOutputType trace = parser.parse(new File(inputFile), true); + + log.info("Parsing finished; in {} seconds", (System.currentTimeMillis() - start) / 1000); + return trace; + } + + private void printStatistics(TracingOutputType trace) { + TraceStatistics statistics = options.isPrintStatExtra() ? TraceStatistics.extra(trace) : TraceStatistics.simple(trace); + log.info("Trace statistics:\n{}", statistics.dump(TraceStatistics.SortBy.SIZE)); + } + + private void applyKeep(TracingOutputType trace) { + List patterns = getPatterns(options.getKeep()); + if (!patterns.isEmpty()) { + List matchingChildren = new ArrayList<>(); + applyKeep(trace.getResult(), matchingChildren, patterns); + + log.info("Keeping {} matching nodes", matchingChildren.size()); + trace.getResult().getPartialResults().clear(); + trace.getResult().getPartialResults().addAll(matchingChildren); + } + } + + private void applyKeep(OperationResultType result, List matchingChildren, List patterns) { + for (OperationResultType child : result.getPartialResults()) { + if (matches(child, patterns)) { + matchingChildren.add(child); + } else { + applyKeep(child, matchingChildren, patterns); + } + } + } + + private void applyKill(TracingOutputType trace) { + List patterns = getPatterns(options.getKill()); + if (!patterns.isEmpty()) { + applyKill(trace.getResult(), patterns); + log.info("Killed {} nodes", killed); + } + } + + private void applyKill(OperationResultType result, List patterns) { + Iterator iterator = result.getPartialResults().iterator(); + while (iterator.hasNext()) { + OperationResultType child = iterator.next(); + if (matches(child, patterns)) { + iterator.remove(); + killed++; + } else { + applyKill(child, patterns); + } + } + } + + private boolean matches(OperationResultType result, List patterns) { + for (Pattern pattern : patterns) { + if (pattern.matcher(result.getOperation()).matches()) { + return true; + } + } + return false; + } + + private List getPatterns(List templates) { + List compiledPatterns = new ArrayList<>(); + for (String template : templates) { + String regex = toRegex(template); + compiledPatterns.add(Pattern.compile(regex)); + } + return compiledPatterns; + } + + private String toRegex(String template) { + return template.replace(".", "\\.").replace("*", ".*"); + } + + private void writeTrace(TracingOutputType trace) throws SchemaException, IOException { + String output = ObjectUtils.defaultIfNull(options.getOutput(), DEFAULT_OUTPUT); + log.info("Starting writing trace to {}", output); + new TraceWriter(context.getPrismContext()).writeTrace(trace, new File(output), true); + log.info("Trace written."); + } + +} diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/TraceStatistics.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/TraceStatistics.java new file mode 100644 index 00000000000..0cab11f663f --- /dev/null +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/TraceStatistics.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2020 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ + +package com.evolveum.midpoint.ninja.action.trace; + +import com.evolveum.midpoint.prism.PrismContext; +import com.evolveum.midpoint.schema.util.OperationResultUtil; +import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.util.exception.SystemException; +import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultType; +import com.evolveum.midpoint.xml.ns._public.common.common_3.TracingOutputType; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * + */ +public class TraceStatistics { + + private final Map operationsMap = new HashMap<>(); + + private final PrismContext prismContext; + private final boolean extra; + + private TraceStatistics(PrismContext prismContext, boolean extra) { + this.prismContext = prismContext; + this.extra = extra; + } + + static TraceStatistics extra(TracingOutputType tracingOutput) { + TraceStatistics traceStatistics = new TraceStatistics(tracingOutput.asPrismContainerValue().getPrismContext(), true); + traceStatistics.update(tracingOutput.getResult()); + return traceStatistics; + } + + static TraceStatistics simple(TracingOutputType tracingOutput) { + TraceStatistics traceStatistics = new TraceStatistics(tracingOutput.asPrismContainerValue().getPrismContext(), false); + traceStatistics.update(tracingOutput.getResult()); + return traceStatistics; + } + + private void update(OperationResultType result) { + operationsMap.compute(result.getOperation(), + (op, info) -> Info.update(result, info, extra, prismContext)); + + result.getPartialResults().forEach(this::update); + } + + @SuppressWarnings("SameParameterValue") + String dump(SortBy sortBy) { + + int maxOp = operationsMap.keySet().stream() + .mapToInt(String::length) + .max().orElse(0); + + String details = operationsMap.entrySet().stream() + .sorted((e1, e2) -> compare(e1, e2, sortBy)) + .map(entry -> String.format("%-" + maxOp + "s %,10d %,15d %,10d", entry.getKey(), + entry.getValue().count, entry.getValue().size, entry.getValue().getAvgSize())) + .collect(Collectors.joining("\n")); + + int nodesCount = operationsMap.values().stream() + .mapToInt(info -> info.count) + .sum(); + + int totalSize = operationsMap.values().stream() + .mapToInt(info -> info.size) + .sum(); + + return "Total nodes: " + nodesCount + "\n" + + (extra ? "Total size: " + totalSize + "\n" : "") + + "\nDetails:\n\n" + details; + } + + private int compare(Map.Entry e1, Map.Entry e2, SortBy sortBy) { + switch (sortBy) { + case OPERATION: + return e1.getKey().compareTo(e2.getKey()); + case COUNT: + return Integer.compare(e2.getValue().count, e1.getValue().count); + case SIZE: + return Integer.compare(e2.getValue().size, e1.getValue().size); + default: + throw new AssertionError(sortBy); + } + } + + private static class Info { + private int count; + private int size; + + private static Info update(OperationResultType result, Info info, boolean extra, PrismContext prismContext) { + if (info == null) { + info = new Info(); + } + info.count++; + if (extra) { + info.size += getSize(result, prismContext); + } + return info; + } + + private static int getSize(OperationResultType result, PrismContext prismContext) { + OperationResultType resultNoChildren = OperationResultUtil.shallowClone(result, false, true, true); + try { + return prismContext.xmlSerializer().serializeRealValue(resultNoChildren).length(); + } catch (SchemaException e) { + throw new SystemException(e); + } + } + + public int getAvgSize() { + return size / count; + } + } + + public enum SortBy { + OPERATION, COUNT, SIZE + } +} diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/impl/Command.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/impl/Command.java index ed729a0ffe3..406cc13b145 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/impl/Command.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/impl/Command.java @@ -7,6 +7,7 @@ package com.evolveum.midpoint.ninja.impl; import com.evolveum.midpoint.ninja.action.*; +import com.evolveum.midpoint.ninja.action.trace.EditTraceAction; import com.evolveum.midpoint.ninja.opts.*; /** @@ -30,7 +31,9 @@ public enum Command { // // TEST("test", TestResourceOptions.class, null, TestResourceRestAction.class), // - KEYS("keys", ListKeysOptions.class, ListKeysRepositoryAction.class, null); + KEYS("keys", ListKeysOptions.class, ListKeysRepositoryAction.class, null), + + TRACE("trace", EditTraceOptions.class, EditTraceAction.class, null); // // TRANSFORM("transform", TransformOptions.class, TransformRepositoryAction.class, null), // diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/impl/NinjaContext.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/impl/NinjaContext.java index d6da82cb298..33894eba043 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/impl/NinjaContext.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/impl/NinjaContext.java @@ -32,7 +32,7 @@ public class NinjaContext { private static final String CTX_NINJA = "classpath:ctx-ninja.xml"; - private static final String[] CTX_MIDPOINT = new String[]{ + private static final String[] CTX_MIDPOINT = new String[] { "classpath:ctx-common.xml", "classpath:ctx-configuration.xml", "classpath:ctx-repository.xml", @@ -40,6 +40,11 @@ public class NinjaContext { "classpath:ctx-audit.xml" }; + private static final String[] CTX_MIDPOINT_NO_REPO = new String[] { + "classpath:ctx-common.xml", + "classpath:ctx-configuration-no-repo.xml" + }; + private JCommander jc; private Log log; @@ -87,7 +92,9 @@ public void setLog(Log log) { } private RepositoryService setupRepositoryViaMidPointHome(ConnectionOptions options) { - log.info("Initializing repository using midpoint home"); + boolean connectRepo = !options.isOffline(); + + log.info("Initializing using midpoint home; {} repository connection", connectRepo ? "with" : "WITHOUT"); System.setProperty(MidpointConfiguration.MIDPOINT_SILENT_PROPERTY, "true"); @@ -107,12 +114,12 @@ private RepositoryService setupRepositoryViaMidPointHome(ConnectionOptions optio GenericXmlApplicationContext ctx = new GenericXmlApplicationContext(); ctx.addBeanFactoryPostProcessor(beanFactory -> beanFactory.addBeanPostProcessor(postprocessor)); ctx.load(CTX_NINJA); - ctx.load(CTX_MIDPOINT); + ctx.load(connectRepo ? CTX_MIDPOINT : CTX_MIDPOINT_NO_REPO); ctx.refresh(); context = ctx; - return context.getBean(REPOSITORY_SERVICE_BEAN, RepositoryService.class); + return connectRepo ? context.getBean(REPOSITORY_SERVICE_BEAN, RepositoryService.class) : null; } public ApplicationContext getApplicationContext() { diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/opts/ConnectionOptions.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/opts/ConnectionOptions.java index 555361487fe..fcb79a2ae75 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/opts/ConnectionOptions.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/opts/ConnectionOptions.java @@ -34,6 +34,8 @@ public class ConnectionOptions { public static final String P_WEBSERVICE = "-w"; public static final String P_WEBSERVICE_LONG = "--webservice"; + public static final String P_OFFLINE_LONG = "--offline"; + @Parameter(names = {P_URL, P_URL_LONG}, validateWith = URIConverter.class, descriptionKey = "connection.url") private String url; @@ -54,6 +56,9 @@ public class ConnectionOptions { @Parameter(names = {P_WEBSERVICE, P_WEBSERVICE_LONG}, descriptionKey = "connection.useWebservice", hidden = true) //todo remove hidden when implementation is done private boolean useWebservice; + @Parameter(names = {P_OFFLINE_LONG}, descriptionKey = "connection.offline") + private boolean offline; + public String getAskPassword() { return askPassword; } @@ -77,4 +82,8 @@ public String getMidpointHome() { public boolean isUseWebservice() { return useWebservice; } + + public boolean isOffline() { + return offline; + } } diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/opts/EditTraceOptions.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/opts/EditTraceOptions.java new file mode 100644 index 00000000000..f53af44fe75 --- /dev/null +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/opts/EditTraceOptions.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2010-2019 Evolveum and contributors + * + * This work is dual-licensed under the Apache License 2.0 + * and European Union Public License. See LICENSE file for details. + */ +package com.evolveum.midpoint.ninja.opts; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import java.util.List; + +/** + * + */ +@Parameters(resourceBundle = "messages", commandDescriptionKey = "editTrace") +public class EditTraceOptions { + + private static final String P_INPUT_LONG = "--input"; + private static final String P_OUTPUT_LONG = "--output"; + private static final String P_PRINT_STAT_LONG = "--print-stat"; + private static final String P_PRINT_STAT_EXTRA_LONG = "--print-stat-extra"; + private static final String P_KILL_LONG = "--kill"; + private static final String P_KEEP_LONG = "--keep"; + + @Parameter(names = { P_INPUT_LONG }, descriptionKey = "editTrace.input") + private String input; + + @Parameter(names = { P_OUTPUT_LONG }, descriptionKey = "editTrace.output") + private String output; + + @Parameter(names = { P_PRINT_STAT_LONG }, descriptionKey = "editTrace.printStat") + private boolean printStat; + + @Parameter(names = { P_PRINT_STAT_EXTRA_LONG }, descriptionKey = "editTrace.printStatExtra") + private boolean printStatExtra; + + @Parameter(names = { P_KILL_LONG }, descriptionKey = "editTrace.kill") + private List kill; + + @Parameter(names = { P_KEEP_LONG }, descriptionKey = "editTrace.keep") + private List keep; + + public String getInput() { + return input; + } + + public String getOutput() { + return output; + } + + public boolean isPrintStat() { + return printStat; + } + + public boolean isPrintStatExtra() { + return printStatExtra; + } + + public List getKill() { + return kill; + } + + public List getKeep() { + return keep; + } +} diff --git a/tools/ninja/src/main/resources/messages.properties b/tools/ninja/src/main/resources/messages.properties index 242da928fea..7f805b6e963 100644 --- a/tools/ninja/src/main/resources/messages.properties +++ b/tools/ninja/src/main/resources/messages.properties @@ -31,6 +31,7 @@ connection.askPassword=Please write rest/jdbc connection password connection.midpointHome=Path to MidPoint home folder. If relative path is specified, it will be translated \ to absolute path. connection.useWebservice=Use model webservice to connect to MidPoint +connection.offline=Do not use repository (local nor via webservice) delete=Delete objects from MidPoint delete.oid=Object oid delete.raw=Raw option @@ -43,7 +44,7 @@ export.output= export.overwrite= export.split= verify=Verify objects in midPoint repository -verify.warn=List of displayed varning categories, e.g. deprecated,plannedRemoval +verify.warn=List of displayed warning categories, e.g. deprecated,plannedRemoval passwordReset=Command will reset password of user specified by oid passwordReset.oid= testResource=Test resource @@ -53,6 +54,13 @@ unlock.oid=User oid listKeys=List keys from keystore listKeys.keyPassword=Key password listKeys.askKeyPassword=Please write key password +editTrace=Edit trace file +editTrace.input=Input trace file +editTrace.output=Output trace file (default is "output.zip") +editTrace.printStat=Print statistics +editTrace.printStatExtra=Print extra statistics i.e. the node size. Takes longer time. +editTrace.kill=Operation results to be removed (with their children). Asterisk is allowed. +editTrace.keep=Operation results to be kept (with their children). Asterisk is allowed. All above them will be removed. schema=DB schema operations schema.test=Validate DB schema schema.init=Initialize DB schema From dd223fa50ef852841715b3d62a03820b79ad56ad Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Fri, 4 Sep 2020 07:10:00 +0200 Subject: [PATCH 11/12] Add missing ninja dependency --- tools/ninja/pom.xml | 4 ++++ .../evolveum/midpoint/ninja/action/trace/EditTraceAction.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/ninja/pom.xml b/tools/ninja/pom.xml index 5646778504e..63f66e698c8 100644 --- a/tools/ninja/pom.xml +++ b/tools/ninja/pom.xml @@ -160,6 +160,10 @@ commons-io commons-io + + org.apache.commons + commons-collections4 + commons-codec commons-codec diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/EditTraceAction.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/EditTraceAction.java index a11143eeabc..b472c092181 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/EditTraceAction.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/trace/EditTraceAction.java @@ -23,7 +23,7 @@ import com.evolveum.midpoint.xml.ns._public.common.common_3.OperationResultType; import com.evolveum.midpoint.xml.ns._public.common.common_3.TracingOutputType; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; /** From 7a8ea4009f810ddd8b625e3e27bee6437532c342 Mon Sep 17 00:00:00 2001 From: lskublik Date: Fri, 4 Sep 2020 07:45:46 +0200 Subject: [PATCH 12/12] modification of audit collection initial objects --- .../285-object-collection-audit.xml | 33 +++++++++++++++++++ .../290-object-collection-audit-errors.xml | 30 ++++++++++++++++- ...-object-collection-audit-modifications.xml | 30 ++++++++++++++++- 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 gui/admin-gui/src/main/resources/initial-objects/285-object-collection-audit.xml diff --git a/gui/admin-gui/src/main/resources/initial-objects/285-object-collection-audit.xml b/gui/admin-gui/src/main/resources/initial-objects/285-object-collection-audit.xml new file mode 100644 index 00000000000..fd0f01dd924 --- /dev/null +++ b/gui/admin-gui/src/main/resources/initial-objects/285-object-collection-audit.xml @@ -0,0 +1,33 @@ + + + + + Audit records + AuditEventRecordType + + + + eventStage + execution + + + timestamp + + + + + + + diff --git a/gui/admin-gui/src/main/resources/initial-objects/290-object-collection-audit-errors.xml b/gui/admin-gui/src/main/resources/initial-objects/290-object-collection-audit-errors.xml index 41aecd26827..77d7db5af5c 100644 --- a/gui/admin-gui/src/main/resources/initial-objects/290-object-collection-audit-errors.xml +++ b/gui/admin-gui/src/main/resources/initial-objects/290-object-collection-audit-errors.xml @@ -7,11 +7,39 @@ --> Error audit records select * from m_audit_event as aer where aer.outcome=3 and aer.eventStage=1 - select * from m_audit_event as aer where aer.eventStage=1 P1D + AuditEventRecordType + + + + eventStage + execution + + + outcome + fatal_error + + + timestamp + + + + + + + + + diff --git a/gui/admin-gui/src/main/resources/initial-objects/300-object-collection-audit-modifications.xml b/gui/admin-gui/src/main/resources/initial-objects/300-object-collection-audit-modifications.xml index 29d303638a3..162f90b53e9 100644 --- a/gui/admin-gui/src/main/resources/initial-objects/300-object-collection-audit-modifications.xml +++ b/gui/admin-gui/src/main/resources/initial-objects/300-object-collection-audit-modifications.xml @@ -7,11 +7,39 @@ --> Modification audit records + AuditEventRecordType + + + + eventStage + execution + + + eventType + modifyObject + + + timestamp + + + + + + + + + select * from m_audit_event as aer where aer.eventType=2 and aer.eventStage=1 - select * from m_audit_event as aer where aer.eventStage=1 P1D