ninja: added exportAudit command + Axiom filter support via % prefix

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/AbstractRepositorySearchAction.java

@@ -39,9 +39,9 @@
 /**
  * Abstract action for all search-based operations, such as export and verify.
  *
- * @author Viliam Repan (lazyman)
+ * @param <O> options class
  */
-public abstract class AbstractRepositorySearchAction<OP extends ExportOptions> extends RepositoryAction<OP> {
+public abstract class AbstractRepositorySearchAction<O extends ExportOptions> extends RepositoryAction<O> {
 
     private static final String DOT_CLASS = AbstractRepositorySearchAction.class.getName() + ".";
 
@@ -91,7 +91,10 @@ public void execute() throws Exception {
         }
 
         executor.shutdown();
-        executor.awaitTermination(NinjaUtils.WAIT_FOR_EXECUTOR_FINISH, TimeUnit.DAYS);
+        boolean awaitResult = executor.awaitTermination(NinjaUtils.WAIT_FOR_EXECUTOR_FINISH, TimeUnit.DAYS);
+        if (!awaitResult) {
+            log.error("Executor did not finish before timeout");
+        }
 
         handleResultOnFinish(operation, "Finished " + getOperationShortName());
     }

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/Action.java

@@ -1,11 +1,13 @@
 /*
- * Copyright (c) 2010-2018 Evolveum and contributors
+ * Copyright (C) 2010-2021 Evolveum and contributors
  *
  * This work is dual-licensed under the Apache License 2.0
  * and European Union Public License. See LICENSE file for details.
  */
 package com.evolveum.midpoint.ninja.action;
 
+import java.util.Objects;
+
 import com.evolveum.midpoint.ninja.impl.LogTarget;
 import com.evolveum.midpoint.ninja.impl.NinjaContext;
 import com.evolveum.midpoint.ninja.opts.ConnectionOptions;
@@ -15,17 +17,19 @@
 import com.evolveum.midpoint.schema.result.OperationResult;
 
 /**
- * Created by Viliam Repan (lazyman).
+ * Base implementation class for action, that is Ninja command.
+ *
+ * @param <O> options class
  */
-public abstract class Action<T> {
+public abstract class Action<O> {
 
     protected Log log;
 
     protected NinjaContext context;
 
-    protected T options;
+    protected O options;
 
-    public void init(NinjaContext context, T options) {
+    public void init(NinjaContext context, O options) {
         this.context = context;
         this.options = options;
 
@@ -34,7 +38,8 @@ public void init(NinjaContext context, T options) {
 
         this.context.setLog(log);
 
-        ConnectionOptions connection = NinjaUtils.getOptions(this.context.getJc(), ConnectionOptions.class);
+        ConnectionOptions connection = Objects.requireNonNull(
+                NinjaUtils.getOptions(this.context.getJc(), ConnectionOptions.class));
         this.context.init(connection);
     }
 

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/RepositoryAction.java

@@ -1,14 +1,16 @@
 /*
- * Copyright (c) 2010-2018 Evolveum and contributors
+ * Copyright (C) 2010-2021 Evolveum and contributors
  *
  * This work is dual-licensed under the Apache License 2.0
  * and European Union Public License. See LICENSE file for details.
  */
 package com.evolveum.midpoint.ninja.action;
 
 /**
- * Created by Viliam Repan (lazyman).
+ * Base implementation class for action (Ninja command) running against the repository.
+ *
+ * @param <O> options class
  */
-public abstract class RepositoryAction<T> extends Action<T> {
+public abstract class RepositoryAction<O> extends Action<O> {
 
 }

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/audit/AuditExportProducerWorker.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2010-2021 Evolveum and contributors
+ *
+ * This work is dual-licensed under the Apache License 2.0
+ * and European Union Public License. See LICENSE file for details.
+ */
+package com.evolveum.midpoint.ninja.action.audit;
+
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+import com.evolveum.midpoint.audit.api.AuditResultHandler;
+import com.evolveum.midpoint.audit.api.AuditService;
+import com.evolveum.midpoint.ninja.action.worker.BaseWorker;
+import com.evolveum.midpoint.ninja.impl.NinjaContext;
+import com.evolveum.midpoint.ninja.impl.NinjaException;
+import com.evolveum.midpoint.ninja.opts.ExportOptions;
+import com.evolveum.midpoint.ninja.util.Log;
+import com.evolveum.midpoint.ninja.util.NinjaUtils;
+import com.evolveum.midpoint.ninja.util.OperationStatus;
+import com.evolveum.midpoint.prism.query.ObjectQuery;
+import com.evolveum.midpoint.schema.GetOperationOptionsBuilder;
+import com.evolveum.midpoint.util.exception.SchemaException;
+import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType;
+
+/**
+ * Producer worker for audit export operation.
+ */
+public class AuditExportProducerWorker extends BaseWorker<ExportOptions, AuditEventRecordType> {
+
+    private final ObjectQuery query;
+
+    public AuditExportProducerWorker(
+            NinjaContext context, ExportOptions options, BlockingQueue<AuditEventRecordType> queue,
+            OperationStatus operation, List<AuditExportProducerWorker> producers, ObjectQuery query) {
+        super(context, options, queue, operation, producers);
+
+        this.query = query;
+    }
+
+    @Override
+    public void run() {
+        Log log = context.getLog();
+
+        try {
+            GetOperationOptionsBuilder optionsBuilder = context.getSchemaService().getOperationOptionsBuilder();
+            if (options.isRaw()) {
+                optionsBuilder = optionsBuilder.raw();
+            }
+
+            optionsBuilder = NinjaUtils.addIncludeOptionsForExport(optionsBuilder, AuditEventRecordType.class);
+
+            AuditResultHandler handler = (object, parentResult) -> {
+                try {
+                    //noinspection unchecked
+                    queue.put(object); // TODO no better way of conversion?
+                } catch (InterruptedException ex) {
+                    log.error("Couldn't queue object {}, reason: {}", ex, object, ex.getMessage());
+                }
+                return true;
+            };
+
+            AuditService auditService = context.getAuditService();
+            auditService.searchObjectsIterative(query, handler, optionsBuilder.build(), operation.getResult());
+        } catch (SchemaException ex) {
+            log.error("Unexpected exception, reason: {}", ex, ex.getMessage());
+        } catch (NinjaException ex) {
+            log.error(ex.getMessage(), ex);
+        } finally {
+            markDone();
+
+            if (isWorkersDone()) {
+                if (!operation.isFinished()) {
+                    operation.producerFinish();
+                }
+            }
+        }
+    }
+}

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/audit/ExportAuditConsumerWorker.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2010-2021 Evolveum and contributors
+ *
+ * This work is dual-licensed under the Apache License 2.0
+ * and European Union Public License. See LICENSE file for details.
+ */
+package com.evolveum.midpoint.ninja.action.audit;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.concurrent.BlockingQueue;
+
+import com.evolveum.midpoint.ninja.action.worker.AbstractWriterConsumerWorker;
+import com.evolveum.midpoint.ninja.impl.NinjaContext;
+import com.evolveum.midpoint.ninja.opts.ExportOptions;
+import com.evolveum.midpoint.ninja.util.NinjaUtils;
+import com.evolveum.midpoint.ninja.util.OperationStatus;
+import com.evolveum.midpoint.prism.PrismSerializer;
+import com.evolveum.midpoint.prism.SerializationOptions;
+import com.evolveum.midpoint.util.exception.SchemaException;
+import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType;
+
+/**
+ * Created by Viliam Repan (lazyman).
+ */
+public class ExportAuditConsumerWorker
+        extends AbstractWriterConsumerWorker<ExportOptions, AuditEventRecordType> {
+
+    private PrismSerializer<String> serializer;
+
+    public ExportAuditConsumerWorker(NinjaContext context,
+            ExportOptions options, BlockingQueue<AuditEventRecordType> queue, OperationStatus operation) {
+        super(context, options, queue, operation);
+    }
+
+    @Override
+    protected void init() {
+        serializer = context.getPrismContext()
+                .xmlSerializer()
+                .options(SerializationOptions.createSerializeForExport().skipContainerIds(options.isSkipContainerIds()));
+    }
+
+    @Override
+    protected String getProlog() {
+        return NinjaUtils.XML_OBJECTS_PREFIX;
+    }
+
+    @Override
+    protected void write(Writer writer, AuditEventRecordType object) throws SchemaException, IOException {
+        String xml = serializer.serialize(object.asPrismContainerValue());
+        writer.write(xml);
+    }
+
+    @Override
+    protected String getEpilog() {
+        return NinjaUtils.XML_OBJECTS_SUFFIX;
+    }
+}

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/audit/ExportAuditRepositoryAction.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) 2010-2021 Evolveum and contributors
+ *
+ * This work is dual-licensed under the Apache License 2.0
+ * and European Union Public License. See LICENSE file for details.
+ */
+package com.evolveum.midpoint.ninja.action.audit;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+
+import com.evolveum.midpoint.ninja.action.AbstractRepositorySearchAction;
+import com.evolveum.midpoint.ninja.action.ExportRepositoryAction;
+import com.evolveum.midpoint.ninja.action.RepositoryAction;
+import com.evolveum.midpoint.ninja.action.worker.ProgressReporterWorker;
+import com.evolveum.midpoint.ninja.impl.LogTarget;
+import com.evolveum.midpoint.ninja.opts.ExportOptions;
+import com.evolveum.midpoint.ninja.util.NinjaUtils;
+import com.evolveum.midpoint.ninja.util.OperationStatus;
+import com.evolveum.midpoint.prism.query.ObjectFilter;
+import com.evolveum.midpoint.prism.query.ObjectQuery;
+import com.evolveum.midpoint.prism.query.QueryFactory;
+import com.evolveum.midpoint.schema.result.OperationResult;
+import com.evolveum.midpoint.util.exception.SchemaException;
+import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventRecordType;
+
+/**
+ * Similar to normal repository {@link ExportRepositoryAction}, but not extended from
+ * {@link AbstractRepositorySearchAction} because we need containers here and objects are quite
+ * deeply embedded in the existing classes.
+ */
+public class ExportAuditRepositoryAction extends RepositoryAction<ExportOptions> {
+
+    private static final int QUEUE_CAPACITY_PER_THREAD = 100;
+    private static final long CONSUMERS_WAIT_FOR_START = 2000L;
+
+    public static final String OPERATION_SHORT_NAME = "exportAudit";
+
+    protected Runnable createConsumer(
+            BlockingQueue<AuditEventRecordType> queue, OperationStatus operation) {
+        return new ExportAuditConsumerWorker(context, options, queue, operation);
+    }
+
+    protected String getOperationName() {
+        return this.getClass().getName() + "." + OPERATION_SHORT_NAME;
+    }
+
+    @Override
+    public void execute() throws Exception {
+        OperationResult result = new OperationResult(getOperationName());
+        OperationStatus operation = new OperationStatus(context, result);
+
+        // "+ 2" will be used for consumer and progress reporter
+        ExecutorService executor = Executors.newFixedThreadPool(options.getMultiThread() + 2);
+
+        BlockingQueue<AuditEventRecordType> queue =
+                new LinkedBlockingQueue<>(QUEUE_CAPACITY_PER_THREAD * options.getMultiThread());
+
+        List<AuditExportProducerWorker> producers = createProducers(queue, operation);
+
+        log.info("Starting " + OPERATION_SHORT_NAME);
+        operation.start();
+
+        // execute as many producers as there are threads for them
+        for (int i = 0; i < producers.size() && i < options.getMultiThread(); i++) {
+            executor.execute(producers.get(i));
+        }
+
+        Thread.sleep(CONSUMERS_WAIT_FOR_START);
+
+        executor.execute(new ProgressReporterWorker<>(context, options, queue, operation));
+
+        Runnable consumer = createConsumer(queue, operation);
+        executor.execute(consumer);
+
+        // execute rest of the producers
+        for (int i = options.getMultiThread(); i < producers.size(); i++) {
+            executor.execute(producers.get(i));
+        }
+
+        executor.shutdown();
+        boolean awaitResult = executor.awaitTermination(NinjaUtils.WAIT_FOR_EXECUTOR_FINISH, TimeUnit.DAYS);
+        if (!awaitResult) {
+            log.error("Executor did not finish before timeout");
+        }
+
+        handleResultOnFinish(operation, "Finished " + OPERATION_SHORT_NAME);
+    }
+
+    @Override
+    public LogTarget getInfoLogTarget() {
+        if (options.getOutput() != null) {
+            return LogTarget.SYSTEM_OUT;
+        }
+
+        return LogTarget.SYSTEM_ERR;
+    }
+
+    private List<AuditExportProducerWorker> createProducers(
+            BlockingQueue<AuditEventRecordType> queue, OperationStatus operation)
+            throws SchemaException, IOException {
+
+        QueryFactory queryFactory = context.getPrismContext().queryFactory();
+        List<AuditExportProducerWorker> producers = new ArrayList<>();
+
+        if (options.getOid() != null) {
+            log.info("OID is ignored for audit export");
+        }
+
+        ObjectFilter filter = NinjaUtils.createObjectFilter(options.getFilter(), context, AuditEventRecordType.class);
+        ObjectQuery query = queryFactory.createQuery(filter);
+
+        producers.add(new AuditExportProducerWorker(context, options, queue, operation, producers, query));
+
+        return producers;
+    }
+}

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/worker/AbstractWriterConsumerWorker.java

@@ -4,7 +4,6 @@
  * This work is dual-licensed under the Apache License 2.0
  * and European Union Public License. See LICENSE file for details.
  */
-
 package com.evolveum.midpoint.ninja.action.worker;
 
 import java.io.IOException;
@@ -23,11 +22,11 @@
 /**
  * Created by Viliam Repan (lazyman).
  */
-public abstract class AbstractWriterConsumerWorker<OP extends ExportOptions, T>
-        extends BaseWorker<OP, T> {
+public abstract class AbstractWriterConsumerWorker<O extends ExportOptions, T>
+        extends BaseWorker<O, T> {
 
     public AbstractWriterConsumerWorker(NinjaContext context,
-            OP options, BlockingQueue<T> queue, OperationStatus operation) {
+            O options, BlockingQueue<T> queue, OperationStatus operation) {
         super(context, options, queue, operation);
     }