Skip to content

Commit

Permalink
ninja: split of audit options from previous hierchy, queue type fixes
Browse files Browse the repository at this point in the history
producer->consumer queue was recently migrated to Containerable, but
some PrismObjects were still forgotten, now it's fixed.
Options for audit import/export inherited unusable options, fixed.
  • Loading branch information
virgo47 committed Dec 10, 2021
1 parent 01c531d commit 9a655d7
Show file tree
Hide file tree
Showing 28 changed files with 273 additions and 122 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import com.evolveum.midpoint.schema.constants.ObjectTypes;
import com.evolveum.midpoint.schema.result.OperationResult;
import com.evolveum.midpoint.util.exception.SchemaException;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceType;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType;

Expand All @@ -52,7 +53,7 @@ public abstract class AbstractRepositorySearchAction<O extends ExportOptions> ex

protected abstract String getOperationShortName();

protected abstract Runnable createConsumer(BlockingQueue<PrismObject<?>> queue, OperationStatus operation);
protected abstract Runnable createConsumer(BlockingQueue<ObjectType> queue, OperationStatus operation);

protected String getOperationName() {
return this.getClass().getName() + "." + getOperationShortName();
Expand All @@ -66,7 +67,7 @@ public void execute() throws Exception {
// "+ 2" will be used for consumer and progress reporter
ExecutorService executor = Executors.newFixedThreadPool(options.getMultiThread() + 2);

BlockingQueue<PrismObject<?>> queue = new LinkedBlockingQueue<>(QUEUE_CAPACITY_PER_THREAD * options.getMultiThread());
BlockingQueue<ObjectType> queue = new LinkedBlockingQueue<>(QUEUE_CAPACITY_PER_THREAD * options.getMultiThread());

List<SearchProducerWorker> producers = createProducers(queue, operation);

Expand All @@ -80,7 +81,7 @@ public void execute() throws Exception {

Thread.sleep(CONSUMERS_WAIT_FOR_START);

executor.execute(new ProgressReporterWorker(context, options, queue, operation));
executor.execute(new ProgressReporterWorker<>(context, options, queue, operation));

Runnable consumer = createConsumer(queue, operation);
executor.execute(consumer);
Expand Down Expand Up @@ -108,7 +109,7 @@ public LogTarget getInfoLogTarget() {
return LogTarget.SYSTEM_ERR;
}

private List<SearchProducerWorker> createProducers(BlockingQueue<PrismObject<?>> queue, OperationStatus operation)
private List<SearchProducerWorker> createProducers(BlockingQueue<ObjectType> queue, OperationStatus operation)
throws SchemaException, IOException {

QueryFactory queryFactory = context.getPrismContext().queryFactory();
Expand Down Expand Up @@ -149,7 +150,7 @@ private List<SearchProducerWorker> createProducers(BlockingQueue<PrismObject<?>>
* performance reasons.
*/
private List<SearchProducerWorker> createProducersForShadows(
NinjaContext context, BlockingQueue<PrismObject<?>> queue,
NinjaContext context, BlockingQueue<ObjectType> queue,
OperationStatus operation, List<SearchProducerWorker> producers, ObjectFilter filter) {

QueryFactory queryFactory = context.getPrismContext().queryFactory();
Expand All @@ -168,7 +169,7 @@ private List<SearchProducerWorker> createProducersForShadows(

List<PrismObject<ResourceType>> list = resultList.getList();
if (list == null || list.isEmpty()) {
shadowProducers.add(createProducer(queue, operation, producers, ObjectTypes.SHADOW, filter));
shadowProducers.add(createShadowProducer(queue, operation, producers, filter));
return shadowProducers;
}

Expand All @@ -182,10 +183,10 @@ private List<SearchProducerWorker> createProducersForShadows(
fullFilter = queryFactory.createAnd(fullFilter, filter);
}

shadowProducers.add(createProducer(queue, operation, producers, ObjectTypes.SHADOW, fullFilter));
shadowProducers.add(createShadowProducer(queue, operation, producers, fullFilter));
}

// all other shadows (no resourceRef or non existing resourceRef)
// all other shadows (no resourceRef or non-existing resourceRef)
List<ObjectFilter> notFilters = new ArrayList<>();
existingResourceRefs.forEach(f -> notFilters.add(queryFactory.createNot(f)));

Expand All @@ -197,11 +198,11 @@ private List<SearchProducerWorker> createProducersForShadows(
fullFilter = queryFactory.createAnd(fullFilter, filter);
}

shadowProducers.add(createProducer(queue, operation, producers, ObjectTypes.SHADOW, fullFilter));
shadowProducers.add(createShadowProducer(queue, operation, producers, fullFilter));
} catch (Exception ex) {
shadowProducers.clear();

shadowProducers.add(createProducer(queue, operation, producers, ObjectTypes.SHADOW, filter));
shadowProducers.add(createShadowProducer(queue, operation, producers, filter));
}

return shadowProducers;
Expand All @@ -221,9 +222,9 @@ private RefFilter createResourceRefFilter(String oid) throws SchemaException {
return prismContext.queryFactory().createReferenceEqual(ShadowType.F_RESOURCE_REF, def, values);
}

private SearchProducerWorker createProducer(BlockingQueue<PrismObject<?>> queue, OperationStatus operation,
List<SearchProducerWorker> producers, ObjectTypes type, ObjectFilter filter) {
private SearchProducerWorker createShadowProducer(BlockingQueue<ObjectType> queue,
OperationStatus operation, List<SearchProducerWorker> producers, ObjectFilter filter) {
ObjectQuery query = context.getPrismContext().queryFactory().createQuery(filter);
return new SearchProducerWorker(context, options, queue, operation, producers, type, query);
return new SearchProducerWorker(context, options, queue, operation, producers, ObjectTypes.SHADOW, query);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import com.evolveum.midpoint.ninja.action.worker.ExportConsumerWorker;
import com.evolveum.midpoint.ninja.opts.ExportOptions;
import com.evolveum.midpoint.ninja.util.OperationStatus;
import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;

/**
* Ninja action realizing "export" command.
Expand All @@ -24,7 +24,7 @@ protected String getOperationShortName() {
}

@Override
protected Runnable createConsumer(BlockingQueue<PrismObject<?>> queue, OperationStatus operation) {
protected Runnable createConsumer(BlockingQueue<ObjectType> queue, OperationStatus operation) {
return new ExportConsumerWorker(context, options, queue, operation);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import com.evolveum.midpoint.ninja.opts.ImportOptions;
import com.evolveum.midpoint.ninja.util.NinjaUtils;
import com.evolveum.midpoint.ninja.util.OperationStatus;
import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.prism.query.InOidFilter;
import com.evolveum.midpoint.prism.query.ObjectFilter;
import com.evolveum.midpoint.schema.result.OperationResult;
Expand All @@ -40,25 +39,25 @@ public void execute() throws Exception {
OperationResult result = new OperationResult(OPERATION_IMPORT);
OperationStatus progress = new OperationStatus(context, result);

BlockingQueue<PrismObject<?>> queue = new LinkedBlockingQueue<>(QUEUE_CAPACITY_PER_THREAD * options.getMultiThread());
BlockingQueue<ObjectType> queue = new LinkedBlockingQueue<>(QUEUE_CAPACITY_PER_THREAD * options.getMultiThread());

// "+ 2" will be used for producer and progress reporter
ExecutorService executor = Executors.newFixedThreadPool(options.getMultiThread() + 2);

ImportProducerWorker producer;
ImportProducerWorker<ObjectType> producer;
if (options.getOid() != null) {
InOidFilter filter = context.getPrismContext().queryFactory().createInOid(options.getOid());
producer = importByFilter(filter, true, queue, progress);
} else {
ObjectFilter filter = NinjaUtils.createObjectFilter(options.getFilter(), context, ObjectType.class); // todo ok? (ObjectType)
ObjectFilter filter = NinjaUtils.createObjectFilter(options.getFilter(), context, ObjectType.class);
producer = importByFilter(filter, false, queue, progress);
}

executor.execute(producer);

Thread.sleep(CONSUMERS_WAIT_FOR_START);

executor.execute(new ProgressReporterWorker(context, options, queue, progress));
executor.execute(new ProgressReporterWorker<>(context, options, queue, progress));

List<ImportRepositoryConsumerWorker> consumers = createConsumers(queue, progress);
consumers.forEach(c -> executor.execute(c));
Expand All @@ -81,12 +80,13 @@ public LogTarget getInfoLogTarget() {
return LogTarget.SYSTEM_ERR;
}

private ImportProducerWorker importByFilter(ObjectFilter filter, boolean stopAfterFound,
BlockingQueue<PrismObject<?>> queue, OperationStatus status) {
return new ImportProducerWorker(context, options, queue, status, filter, stopAfterFound);
private ImportProducerWorker<ObjectType> importByFilter(ObjectFilter filter,
boolean stopAfterFound, BlockingQueue<ObjectType> queue, OperationStatus status) {
return new ImportProducerWorker<>(context, options, queue, status, filter, stopAfterFound);
}

private List<ImportRepositoryConsumerWorker> createConsumers(BlockingQueue<PrismObject<?>> queue, OperationStatus operation) {
private List<ImportRepositoryConsumerWorker> createConsumers(
BlockingQueue<ObjectType> queue, OperationStatus operation) {
List<ImportRepositoryConsumerWorker> consumers = new ArrayList<>();

for (int i = 0; i < options.getMultiThread(); i++) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import com.evolveum.midpoint.ninja.action.worker.VerifyConsumerWorker;
import com.evolveum.midpoint.ninja.opts.VerifyOptions;
import com.evolveum.midpoint.ninja.util.OperationStatus;
import com.evolveum.midpoint.prism.PrismObject;
import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;

/**
* Created by Viliam Repan (lazyman).
Expand All @@ -24,7 +24,7 @@ protected String getOperationShortName() {
}

@Override
protected Runnable createConsumer(BlockingQueue<PrismObject<?>> queue, OperationStatus operation) {
protected Runnable createConsumer(BlockingQueue<ObjectType> queue, OperationStatus operation) {
return new VerifyConsumerWorker(context, options, queue, operation);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright (C) 2010-2021 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/
package com.evolveum.midpoint.ninja.action.audit;

import com.beust.jcommander.Parameter;

import com.evolveum.midpoint.ninja.util.FileReference;
import com.evolveum.midpoint.ninja.util.FileReferenceConverter;

public class BaseAuditImportExportOptions {

public static final String P_FILTER = "-f";
public static final String P_FILTER_LONG = "--filter";

public static final String P_ZIP = "-z";
public static final String P_ZIP_LONG = "--zip";

public static final String P_MULTI_THREAD = "-l";
public static final String P_MULTI_THREAD_LONG = "--multi-thread";

@Parameter(names = { P_FILTER, P_FILTER_LONG }, descriptionKey = "base.filter",
converter = FileReferenceConverter.class, validateWith = FileReferenceConverter.class)
private FileReference filter;

@Parameter(names = { P_ZIP, P_ZIP_LONG }, descriptionKey = "baseImportExport.zip")
private boolean zip;

@Parameter(names = { P_MULTI_THREAD, P_MULTI_THREAD_LONG }, descriptionKey = "baseImportExport.multiThread")
private int multiThread = 1;

public FileReference getFilter() {
return filter;
}

public boolean isZip() {
return zip;
}

public int getMultiThread() {
return multiThread;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

import com.evolveum.midpoint.ninja.action.worker.AbstractWriterConsumerWorker;
import com.evolveum.midpoint.ninja.impl.NinjaContext;
import com.evolveum.midpoint.ninja.opts.ExportOptions;
import com.evolveum.midpoint.ninja.util.NinjaUtils;
import com.evolveum.midpoint.ninja.util.OperationStatus;
import com.evolveum.midpoint.prism.PrismSerializer;
Expand All @@ -24,12 +23,12 @@
* Consumer writing exported audit events to the writer (stdout or file).
*/
public class ExportAuditConsumerWorker
extends AbstractWriterConsumerWorker<ExportOptions, AuditEventRecordType> {
extends AbstractWriterConsumerWorker<ExportAuditOptions, AuditEventRecordType> {

private PrismSerializer<String> serializer;

public ExportAuditConsumerWorker(NinjaContext context,
ExportOptions options, BlockingQueue<AuditEventRecordType> queue, OperationStatus operation) {
ExportAuditOptions options, BlockingQueue<AuditEventRecordType> queue, OperationStatus operation) {
super(context, options, queue, operation);
}

Expand All @@ -41,7 +40,7 @@ protected void init() {
// TODO: This does not help with RawType: (parsed:ObjectReferenceType) for which
// the names still go only to the comments (ignored by the import, obviously).
.serializeReferenceNames(true)
.skipContainerIds(options.isSkipContainerIds()));
.skipContainerIds(true));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Copyright (C) 2010-2021 Evolveum and contributors
*
* This work is dual-licensed under the Apache License 2.0
* and European Union Public License. See LICENSE file for details.
*/
package com.evolveum.midpoint.ninja.action.audit;

import java.io.File;

import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;

import com.evolveum.midpoint.ninja.opts.BasicExportOptions;

@Parameters(resourceBundle = "messages", commandDescriptionKey = "exportAudit")
public class ExportAuditOptions extends BaseAuditImportExportOptions implements BasicExportOptions {

public static final String P_OUTPUT = "-O";
public static final String P_OUTPUT_LONG = "--output";

public static final String P_OVERWRITE = "-ow";
public static final String P_OVERWRITE_LONG = "--overwrite";

@Parameter(names = { P_OUTPUT, P_OUTPUT_LONG }, descriptionKey = "export.output")
private File output;

@Parameter(names = { P_OVERWRITE, P_OVERWRITE_LONG }, descriptionKey = "export.overwrite")
private boolean overwrite;

public File getOutput() {
return output;
}

public boolean isOverwrite() {
return overwrite;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import com.evolveum.midpoint.ninja.action.worker.BaseWorker;
import com.evolveum.midpoint.ninja.impl.NinjaContext;
import com.evolveum.midpoint.ninja.impl.NinjaException;
import com.evolveum.midpoint.ninja.opts.ExportOptions;
import com.evolveum.midpoint.ninja.util.Log;
import com.evolveum.midpoint.ninja.util.NinjaUtils;
import com.evolveum.midpoint.ninja.util.OperationStatus;
Expand All @@ -26,12 +25,12 @@
/**
* Producer worker for audit export operation.
*/
public class ExportAuditProducerWorker extends BaseWorker<ExportOptions, AuditEventRecordType> {
public class ExportAuditProducerWorker extends BaseWorker<ExportAuditOptions, AuditEventRecordType> {

private final ObjectQuery query;

public ExportAuditProducerWorker(
NinjaContext context, ExportOptions options, BlockingQueue<AuditEventRecordType> queue,
NinjaContext context, ExportAuditOptions options, BlockingQueue<AuditEventRecordType> queue,
OperationStatus operation, List<ExportAuditProducerWorker> producers, ObjectQuery query) {
super(context, options, queue, operation, producers);

Expand All @@ -44,9 +43,6 @@ public void run() {

try {
GetOperationOptionsBuilder optionsBuilder = context.getSchemaService().getOperationOptionsBuilder();
if (options.isRaw()) {
optionsBuilder = optionsBuilder.raw();
}

optionsBuilder = NinjaUtils.addIncludeOptionsForExport(optionsBuilder, AuditEventRecordType.class);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
import com.evolveum.midpoint.ninja.action.RepositoryAction;
import com.evolveum.midpoint.ninja.action.worker.ProgressReporterWorker;
import com.evolveum.midpoint.ninja.impl.LogTarget;
import com.evolveum.midpoint.ninja.opts.ExportOptions;
import com.evolveum.midpoint.ninja.util.NinjaUtils;
import com.evolveum.midpoint.ninja.util.OperationStatus;
import com.evolveum.midpoint.prism.query.ObjectFilter;
Expand All @@ -31,7 +30,7 @@
* {@link AbstractRepositorySearchAction} because we need containers here and objects are quite
* deeply embedded in the existing classes.
*/
public class ExportAuditRepositoryAction extends RepositoryAction<ExportOptions> {
public class ExportAuditRepositoryAction extends RepositoryAction<ExportAuditOptions> {

private static final int QUEUE_CAPACITY_PER_THREAD = 100;
private static final long CONSUMERS_WAIT_FOR_START = 2000L;
Expand Down Expand Up @@ -102,10 +101,6 @@ private List<ExportAuditProducerWorker> createProducers(
QueryFactory queryFactory = context.getPrismContext().queryFactory();
List<ExportAuditProducerWorker> producers = new ArrayList<>();

if (options.getOid() != null) {
log.info("OID is ignored for audit export");
}

ObjectFilter filter = NinjaUtils.createObjectFilter(options.getFilter(), context, AuditEventRecordType.class);
ObjectQuery query = queryFactory.createQuery(filter);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import com.evolveum.midpoint.audit.api.AuditService;
import com.evolveum.midpoint.ninja.action.worker.BaseWorker;
import com.evolveum.midpoint.ninja.impl.NinjaContext;
import com.evolveum.midpoint.ninja.opts.ImportOptions;
import com.evolveum.midpoint.ninja.util.OperationStatus;
import com.evolveum.midpoint.prism.crypto.Protector;
import com.evolveum.midpoint.schema.result.OperationResult;
Expand All @@ -25,10 +24,10 @@
/**
* Consumer importing audit events to the database.
*/
public class ImportAuditConsumerWorker extends BaseWorker<ImportOptions, AuditEventRecordType> {
public class ImportAuditConsumerWorker extends BaseWorker<ImportAuditOptions, AuditEventRecordType> {

public ImportAuditConsumerWorker(
NinjaContext context, ImportOptions options, BlockingQueue<AuditEventRecordType> queue,
NinjaContext context, ImportAuditOptions options, BlockingQueue<AuditEventRecordType> queue,
OperationStatus operation, List<ImportAuditConsumerWorker> consumers) {
super(context, options, queue, operation, consumers);
}
Expand Down

0 comments on commit 9a655d7

Please sign in to comment.