Skip to content

Commit

Permalink
MID-8842 ninja - initial objects action improvements + reporting. Bas…
Browse files Browse the repository at this point in the history 
…ic merge for some types inplace now.
  • Loading branch information
@1azyman
1azyman committed Sep 7, 2023
1 parent fe8c9f6 commit e69b755
Show file tree
Hide file tree
Showing 5 changed files with 233 additions and 39 deletions.
33 changes: 33 additions & 0 deletions ...schema/src/main/java/com/evolveum/midpoint/schema/merger/AdminGuiConfigurationMerger.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2010-2023 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/

package com.evolveum.midpoint.schema.merger;

import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import com.evolveum.midpoint.prism.PrismContainer;
import com.evolveum.midpoint.util.exception.ConfigurationException;
import com.evolveum.midpoint.util.exception.SchemaException;
import com.evolveum.midpoint.xml.ns._public.common.common_3.AdminGuiConfigurationType;

public class AdminGuiConfigurationMerger extends BaseItemMerger<PrismContainer<AdminGuiConfigurationType>> {

public AdminGuiConfigurationMerger(@Nullable OriginMarker originMarker) {
super(originMarker);
}

@Override
protected void mergeInternal(
@NotNull PrismContainer<AdminGuiConfigurationType> target,
@NotNull PrismContainer<AdminGuiConfigurationType> source)
throws ConfigurationException, SchemaException {

AdminGuiConfigurationMergeManager manager = new AdminGuiConfigurationMergeManager();

}
}
53 changes: 53 additions & 0 deletions ...schema/src/main/java/com/evolveum/midpoint/schema/merger/object/ObjectMergeOperation.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
* Copyright (C) 2010-2023 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/

package com.evolveum.midpoint.schema.merger.object;

import static java.util.Map.entry;

import java.lang.reflect.InvocationTargetException;
import java.util.Map;

import org.apache.commons.lang3.Validate;

import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.schema.merger.BaseMergeOperation;
import com.evolveum.midpoint.util.exception.ConfigurationException;
import com.evolveum.midpoint.util.exception.SchemaException;
import com.evolveum.midpoint.util.exception.SystemException;
import com.evolveum.midpoint.xml.ns._public.common.common_3.LookupTableType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;

public class ObjectMergeOperation {

public static final Map<Class<? extends ObjectType>, Class<? extends BaseMergeOperation>> MERGE_OPERATIONS = Map.ofEntries(
entry(LookupTableType.class, LookupTableMergeOperation.class)
);

public static <O extends ObjectType> boolean hasMergeOperationFor(PrismObject<O> target) {
Class<?> type = target.getCompileTimeClass();

return MERGE_OPERATIONS.containsKey(type);
}

public static <O extends ObjectType> void merge(
PrismObject<O> target, PrismObject<O> source) throws ConfigurationException, SchemaException {
Class<?> type = target.getCompileTimeClass();

Class<? extends BaseMergeOperation> clazz = MERGE_OPERATIONS.get(type);
Validate.notNull(clazz, "Merge operation not available for type " + type.getName());

try {
BaseMergeOperation<O> operation = clazz.getConstructor(type, type)
.newInstance(target.asObjectable(), source.asObjectable());
operation.execute();
} catch (NoSuchMethodException | SecurityException | InstantiationException |
IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) {
throw new SystemException("Couldn't merge objects", ex);
}
}
}
158 changes: 119 additions & 39 deletions ...src/main/java/com/evolveum/midpoint/ninja/action/upgrade/action/InitialObjectsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,34 +7,44 @@

package com.evolveum.midpoint.ninja.action.upgrade.action;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;

import com.evolveum.midpoint.ninja.action.Action;
import com.evolveum.midpoint.ninja.action.ActionResult;
import com.evolveum.midpoint.ninja.impl.LogTarget;
import com.evolveum.midpoint.ninja.util.ConsoleFormat;
import com.evolveum.midpoint.ninja.util.NinjaUtils;
import com.evolveum.midpoint.prism.PrismContext;
import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.prism.delta.ObjectDelta;
import com.evolveum.midpoint.repo.api.RepoAddOptions;
import com.evolveum.midpoint.repo.api.RepositoryService;
import com.evolveum.midpoint.schema.DeltaConvertor;
import com.evolveum.midpoint.schema.constants.SchemaConstants;
import com.evolveum.midpoint.schema.merger.object.ObjectMergeOperation;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.util.MiscUtil;
import com.evolveum.midpoint.util.exception.ConfigurationException;
import com.evolveum.midpoint.util.exception.ObjectAlreadyExistsException;
import com.evolveum.midpoint.util.exception.ObjectNotFoundException;
import com.evolveum.midpoint.util.exception.SchemaException;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.TriggerType;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;

import java.io.File;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
import com.evolveum.prism.xml.ns._public.types_3.ObjectDeltaType;

// todo action should write XML + maybe csv? for review
public class InitialObjectsAction extends Action<InitialObjectsOptions, ActionResult<InitialObjectsResult>> {
Expand All @@ -46,34 +56,59 @@ public String getOperationName() {
return "initial objects";
}

@Override
public LogTarget getLogTarget() {
return options.getOutput() != null ? LogTarget.SYSTEM_ERR : LogTarget.SYSTEM_OUT;
}

@Override
public ActionResult<InitialObjectsResult> execute() throws Exception {
InitialObjectsResult actionResult = new InitialObjectsResult();

OperationResult result = new OperationResult("Initial objects update");

List<Resource> resources = new ArrayList<>();
List<File> files = options.getFiles();
if (files != null && !files.isEmpty()) {
for (File file : options.getFiles()) {
if (file.isDirectory()) {
FileUtils.listFiles(file, new String[] { "xml" }, true)
.forEach(f -> resources.add(new FileSystemResource(f)));
} else {
resources.add(new org.springframework.core.io.FileSystemResource(file));
Writer writer = null;
try {
if (options.isReport()) {
writer = NinjaUtils.createWriter(
options.getOutput(), context.getCharset(), options.isZip(), options.isOverwrite(), context.out);
writer.write(NinjaUtils.XML_DELTAS_PREFIX);
}

OperationResult result = new OperationResult("Initial objects update");

List<Resource> resources = new ArrayList<>();
List<File> files = options.getFiles();
if (files != null && !files.isEmpty()) {
for (File file : options.getFiles()) {
if (file.isDirectory()) {
FileUtils.listFiles(file, new String[] { "xml" }, true)
.forEach(f -> resources.add(new FileSystemResource(f)));
} else {
resources.add(new org.springframework.core.io.FileSystemResource(file));
}
}
} else {
Resource[] array = new PathMatchingResourcePatternResolver().getResources(INITIAL_OBJECTS_RESOURCE_PATTERN);
resources.addAll(Arrays.asList(array));
}
} else {
Resource[] array = new PathMatchingResourcePatternResolver().getResources(INITIAL_OBJECTS_RESOURCE_PATTERN);
resources.addAll(Arrays.asList(array));
}

Collections.sort(resources, Comparator.comparing(Resource::getFilename));
resources.sort(Comparator.comparing(Resource::getFilename));

for (Resource resource : resources) {
actionResult.incrementTotal();
for (Resource resource : resources) {
actionResult.incrementTotal();

processFile(resource, result, actionResult, writer);
}
} finally {
if (writer != null) {
writer.write(NinjaUtils.XML_DELTAS_SUFFIX);

processFile(resource, result, actionResult);
if (options.getOutput() != null) {

// todo this should be handled better, not manually on multiple places
// we don't want to close stdout, e.g. only if we were writing to file
IOUtils.closeQuietly(writer);
}
}
}

int status = actionResult.getError() == 0 ? 0 : 1;
Expand All @@ -92,7 +127,7 @@ public ActionResult<InitialObjectsResult> execute() throws Exception {
}

private <O extends ObjectType> void processFile(
Resource resource, OperationResult parentResult, InitialObjectsResult actionResult) {
Resource resource, OperationResult parentResult, InitialObjectsResult actionResult, Writer writer) {

OperationResult result = parentResult.createSubresult("Process file");

Expand All @@ -114,9 +149,9 @@ private <O extends ObjectType> void processFile(

if (existing == null) {
// we'll just import object, since it's new one
addObject(object, result, actionResult);
addObject(object, result, actionResult, writer);
} else {
mergeObject(object, existing, result, actionResult);
mergeObject(object, existing, result, actionResult, writer);
}
} catch (Exception ex) {
log.error("Unexpected exception occurred processing file {}", ex, resource.getFilename());
Expand All @@ -125,23 +160,30 @@ private <O extends ObjectType> void processFile(
}

private <O extends ObjectType> void mergeObject(
PrismObject<O> initial, PrismObject<O> existing, OperationResult result, InitialObjectsResult actionResult) {
PrismObject<O> initial, PrismObject<O> existing, OperationResult result, InitialObjectsResult actionResult, Writer writer)
throws SchemaException, ConfigurationException, IOException {

log.debug("Merging object {}", NinjaUtils.printObjectNameOidAndType(existing));

final PrismObject<O> merged = existing.clone();

// todo do merge
boolean wasMerged = mergeObject(merged, initial);
if (!wasMerged) {
log.warn("Couldn't merge object {}, skipping", NinjaUtils.printObjectNameOidAndType(existing));
actionResult.incrementError();
}

addTrigger(existing);
ObjectDelta<O> delta = merged.diff(existing);
// addTrigger(existing);
ObjectDelta<O> delta = existing.diff(merged);
if (delta.isEmpty()) {
log.debug("Skipping object update, delta is empty");

actionResult.incrementUnchanged();
return;
}

reportDelta(delta, writer);

try {
log.debug(
"Updating object {} in repository {}",
Expand All @@ -161,6 +203,21 @@ private <O extends ObjectType> void mergeObject(
}
}

private <O extends ObjectType> boolean mergeObject(PrismObject<O> target, PrismObject<O> source)
throws SchemaException, ConfigurationException {

if (target.equivalent(source)) {
return true;
}

if (!ObjectMergeOperation.hasMergeOperationFor(target)) {
return false;
}

ObjectMergeOperation.merge(target, source);
return true;
}

/**
* @deprecated This is just a hack to trigger recompute after midpoint is started. TODO FIXME fix this
*/
Expand All @@ -172,15 +229,38 @@ private <O extends ObjectType> void addTrigger(PrismObject<O> object) {
object.asObjectable().trigger(trigger);
}

private <O extends ObjectType> void reportAddDelta(PrismObject<O> object, Writer writer) throws SchemaException, IOException {
ObjectDelta<O> delta = context.getPrismContext().deltaFactory()
.object()
.createEmptyAddDelta(object.getCompileTimeClass(), object.getOid());
delta.setObjectToAdd(object);

reportDelta(delta, writer);
}

private <O extends ObjectType> void reportDelta(ObjectDelta<O> delta, Writer writer) throws SchemaException, IOException {
if (writer == null || !options.isReport()) {
return;
}

ObjectDeltaType deltaType = DeltaConvertor.toObjectDeltaType(delta);
String xml = context.getPrismContext().xmlSerializer()
.serializeRealValue(deltaType, NinjaUtils.DELTA_LIST_DELTA);
writer.write(xml);
}

private <O extends ObjectType> void addObject(
PrismObject<O> object, OperationResult result, InitialObjectsResult actionResult) {
PrismObject<O> object, OperationResult result, InitialObjectsResult actionResult, Writer writer)
throws SchemaException, IOException {

reportAddDelta(object, writer);

if (!options.isForceAdd()) {
log.debug("Skipping object add, force-add options is not set, object will be correctly added during midpoint startup.");
return;
}

addTrigger(object);
// addTrigger(object);

try {
log.debug(
Expand Down
26 changes: 26 additions & 0 deletions ...rc/main/java/com/evolveum/midpoint/ninja/action/upgrade/action/InitialObjectsOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ public class InitialObjectsOptions {
private static final String P_DRY_RUN = "--dry-run";
public static final String P_OUTPUT = "-o";
public static final String P_OUTPUT_LONG = "--output";
public static final String P_OVERWRITE = "-O";
public static final String P_OVERWRITE_LONG = "--overwrite";
public static final String P_ZIP = "-z";
public static final String P_ZIP_LONG = "--zip";
public static final String P_REPORT = "-r";
public static final String P_REPORT_LONG = "--report";
public static final String P_FORCE_ADD = "--force-add";
Expand All @@ -40,6 +44,12 @@ public class InitialObjectsOptions {
@Parameter(names = { P_REPORT, P_REPORT_LONG }, descriptionKey = "initialObjects.report")
private boolean report;

@Parameter(names = { P_OVERWRITE, P_OVERWRITE_LONG }, descriptionKey = "initialObjects.overwrite")
private boolean overwrite;

@Parameter(names = {P_ZIP, P_ZIP_LONG}, descriptionKey = "initialObjects.zip")
private boolean zip;

public boolean isDryRun() {
return dryRun;
}
Expand Down Expand Up @@ -79,4 +89,20 @@ public boolean isForceAdd() {
public void setForceAdd(boolean forceAdd) {
this.forceAdd = forceAdd;
}

public boolean isOverwrite() {
return overwrite;
}

public void setOverwrite(boolean overwrite) {
this.overwrite = overwrite;
}

public boolean isZip() {
return zip;
}

public void setZip(boolean zip) {
this.zip = zip;
}
}
2 changes: 2 additions & 0 deletions tools/ninja/src/main/resources/messages.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,3 +155,5 @@ initialObjects.report=If true, create report XML file that will contain all delt
initialObjects.files=Files/Directories for initial objects. Directories will be recursively searched for XML files. \
If this option is not specified, default initial objects set bundled withing ninja will be used. Files will be sorted by name.
initialObjects.forceAdd=Force add non yet existing initial objects. By default, objects will be added during midpoint startup.
initialObjects.overwrite=Overwrite output file if it exists.
initialObjects.zip=Use zip/unzip compression for output file.

0 comments on commit e69b755

Please sign in to comment.