Merge branch 'feature/ninja-audit' into support-4.4

repo/audit-api/src/main/java/com/evolveum/midpoint/audit/api/AuditService.java

@@ -25,7 +25,9 @@
 import com.evolveum.midpoint.xml.ns._public.common.common_3.SystemConfigurationAuditType;
 
 /**
- * @author semancik
+ * Service contract for audit - this can actually represent multiple audit services.
+ * This is implementation independent, but some features may not be supported by all implementations.
+ * For instance, {@link #supportsRetrieval()} indicates whether audit supports searching, or just storing.
  */
 public interface AuditService {
 
@@ -54,6 +56,7 @@ public interface AuditService {
      * Used for audit import functionality.
      * This is a low-level audit method that does not process provided record at all.
      */
+    @Experimental
     void audit(AuditEventRecordType record, OperationResult parentResult);
 
     /**

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/AbstractRepositorySearchAction.java

@@ -33,6 +33,7 @@
 import com.evolveum.midpoint.schema.constants.ObjectTypes;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.util.exception.SchemaException;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ResourceType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ShadowType;
 
@@ -52,7 +53,7 @@ public abstract class AbstractRepositorySearchAction<O extends ExportOptions> ex
 
     protected abstract String getOperationShortName();
 
-    protected abstract Runnable createConsumer(BlockingQueue<PrismObject<?>> queue, OperationStatus operation);
+    protected abstract Runnable createConsumer(BlockingQueue<ObjectType> queue, OperationStatus operation);
 
     protected String getOperationName() {
         return this.getClass().getName() + "." + getOperationShortName();
@@ -66,7 +67,7 @@ public void execute() throws Exception {
         // "+ 2" will be used for consumer and progress reporter
         ExecutorService executor = Executors.newFixedThreadPool(options.getMultiThread() + 2);
 
-        BlockingQueue<PrismObject<?>> queue = new LinkedBlockingQueue<>(QUEUE_CAPACITY_PER_THREAD * options.getMultiThread());
+        BlockingQueue<ObjectType> queue = new LinkedBlockingQueue<>(QUEUE_CAPACITY_PER_THREAD * options.getMultiThread());
 
         List<SearchProducerWorker> producers = createProducers(queue, operation);
 
@@ -80,7 +81,7 @@ public void execute() throws Exception {
 
         Thread.sleep(CONSUMERS_WAIT_FOR_START);
 
-        executor.execute(new ProgressReporterWorker(context, options, queue, operation));
+        executor.execute(new ProgressReporterWorker<>(context, options, queue, operation));
 
         Runnable consumer = createConsumer(queue, operation);
         executor.execute(consumer);
@@ -108,7 +109,7 @@ public LogTarget getInfoLogTarget() {
         return LogTarget.SYSTEM_ERR;
     }
 
-    private List<SearchProducerWorker> createProducers(BlockingQueue<PrismObject<?>> queue, OperationStatus operation)
+    private List<SearchProducerWorker> createProducers(BlockingQueue<ObjectType> queue, OperationStatus operation)
             throws SchemaException, IOException {
 
         QueryFactory queryFactory = context.getPrismContext().queryFactory();
@@ -149,7 +150,7 @@ private List<SearchProducerWorker> createProducers(BlockingQueue<PrismObject<?>>
      * performance reasons.
      */
     private List<SearchProducerWorker> createProducersForShadows(
-            NinjaContext context, BlockingQueue<PrismObject<?>> queue,
+            NinjaContext context, BlockingQueue<ObjectType> queue,
             OperationStatus operation, List<SearchProducerWorker> producers, ObjectFilter filter) {
 
         QueryFactory queryFactory = context.getPrismContext().queryFactory();
@@ -168,7 +169,7 @@ private List<SearchProducerWorker> createProducersForShadows(
 
             List<PrismObject<ResourceType>> list = resultList.getList();
             if (list == null || list.isEmpty()) {
-                shadowProducers.add(createProducer(queue, operation, producers, ObjectTypes.SHADOW, filter));
+                shadowProducers.add(createShadowProducer(queue, operation, producers, filter));
                 return shadowProducers;
             }
 
@@ -182,10 +183,10 @@ private List<SearchProducerWorker> createProducersForShadows(
                     fullFilter = queryFactory.createAnd(fullFilter, filter);
                 }
 
-                shadowProducers.add(createProducer(queue, operation, producers, ObjectTypes.SHADOW, fullFilter));
+                shadowProducers.add(createShadowProducer(queue, operation, producers, fullFilter));
             }
 
-            // all other shadows (no resourceRef or non existing resourceRef)
+            // all other shadows (no resourceRef or non-existing resourceRef)
             List<ObjectFilter> notFilters = new ArrayList<>();
             existingResourceRefs.forEach(f -> notFilters.add(queryFactory.createNot(f)));
 
@@ -197,11 +198,11 @@ private List<SearchProducerWorker> createProducersForShadows(
                 fullFilter = queryFactory.createAnd(fullFilter, filter);
             }
 
-            shadowProducers.add(createProducer(queue, operation, producers, ObjectTypes.SHADOW, fullFilter));
+            shadowProducers.add(createShadowProducer(queue, operation, producers, fullFilter));
         } catch (Exception ex) {
             shadowProducers.clear();
 
-            shadowProducers.add(createProducer(queue, operation, producers, ObjectTypes.SHADOW, filter));
+            shadowProducers.add(createShadowProducer(queue, operation, producers, filter));
         }
 
         return shadowProducers;
@@ -221,9 +222,9 @@ private RefFilter createResourceRefFilter(String oid) throws SchemaException {
         return prismContext.queryFactory().createReferenceEqual(ShadowType.F_RESOURCE_REF, def, values);
     }
 
-    private SearchProducerWorker createProducer(BlockingQueue<PrismObject<?>> queue, OperationStatus operation,
-            List<SearchProducerWorker> producers, ObjectTypes type, ObjectFilter filter) {
+    private SearchProducerWorker createShadowProducer(BlockingQueue<ObjectType> queue,
+            OperationStatus operation, List<SearchProducerWorker> producers, ObjectFilter filter) {
         ObjectQuery query = context.getPrismContext().queryFactory().createQuery(filter);
-        return new SearchProducerWorker(context, options, queue, operation, producers, type, query);
+        return new SearchProducerWorker(context, options, queue, operation, producers, ObjectTypes.SHADOW, query);
     }
 }

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/DeleteRepositoryAction.java

@@ -1,11 +1,20 @@
 /*
- * Copyright (c) 2010-2019 Evolveum and contributors
+ * Copyright (C) 2010-2021 Evolveum and contributors
  *
  * This work is dual-licensed under the Apache License 2.0
  * and European Union Public License. See LICENSE file for details.
  */
 package com.evolveum.midpoint.ninja.action;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.commons.lang3.StringUtils;
+
 import com.evolveum.midpoint.ninja.opts.DeleteOptions;
 import com.evolveum.midpoint.ninja.util.NinjaUtils;
 import com.evolveum.midpoint.ninja.util.OperationStatus;
@@ -22,14 +31,6 @@
 import com.evolveum.midpoint.util.exception.ObjectNotFoundException;
 import com.evolveum.midpoint.util.exception.SchemaException;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.lang.reflect.Modifier;
-import java.util.ArrayList;
-import java.util.Collection;
 
 /**
  * Created by Viliam Repan (lazyman).
@@ -57,15 +58,15 @@ public void execute() throws Exception {
         }
     }
 
-    private void deleteByOid() throws SchemaException, IOException {
+    private void deleteByOid() throws SchemaException {
         QueryFactory queryFactory = context.getPrismContext().queryFactory();
         InOidFilter filter = queryFactory.createInOid(options.getOid());
         ObjectQuery query = queryFactory.createQuery(filter);
 
         deleteByFilter(query);
     }
 
-    private void deleteByFilter(ObjectQuery query) throws SchemaException, IOException {
+    private void deleteByFilter(ObjectQuery query) throws SchemaException {
         OperationResult result = new OperationResult(OPERATION_DELETE);
 
         OperationStatus operation = new OperationStatus(context, result);
@@ -86,14 +87,14 @@ private void deleteByFilter(ObjectQuery query) throws SchemaException, IOExcepti
             }
         }
 
+        operation.finish();
         handleResultOnFinish(operation, "Delete finished");
     }
 
     private void deleteByFilter(ObjectTypes type, ObjectQuery query, OperationStatus operation, OperationResult result)
             throws SchemaException {
 
-        ResultHandler handler = (prismObject, operationResult) -> {
-
+        ResultHandler<?> handler = (prismObject, operationResult) -> {
             try {
                 State state = options.isAsk() ? askForState(prismObject) : State.DELETE;
 
@@ -123,7 +124,7 @@ private void deleteByFilter(ObjectTypes type, ObjectQuery query, OperationStatus
 
         Collection<SelectorOptions<GetOperationOptions>> opts = new ArrayList<>();
         if (options.isRaw()) {
-            opts.add(new SelectorOptions(GetOperationOptions.createRaw()));
+            opts.add(new SelectorOptions<>(GetOperationOptions.createRaw()));
         }
 
         RepositoryService repository = context.getRepository();
@@ -144,15 +145,15 @@ private State askForState(PrismObject object) throws IOException {
 
                 strState = strState.toLowerCase();
 
-                if ("y".equals(strState) || "yes".equals(strState)) {
+                if ("y".equalsIgnoreCase(strState) || "yes".equalsIgnoreCase(strState)) {
                     state = State.DELETE;
                 }
 
-                if ("n".equals(strState) || "no".equals(strState)) {
+                if ("n".equalsIgnoreCase(strState) || "no".equalsIgnoreCase(strState)) {
                     state = State.SKIP;
                 }
 
-                if ("c".equals(strState) || "cancel".equals(strState)) {
+                if ("c".equalsIgnoreCase(strState) || "cancel".equalsIgnoreCase(strState)) {
                     state = State.STOP;
                 }
             }

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/ExportRepositoryAction.java

@@ -11,7 +11,7 @@
 import com.evolveum.midpoint.ninja.action.worker.ExportConsumerWorker;
 import com.evolveum.midpoint.ninja.opts.ExportOptions;
 import com.evolveum.midpoint.ninja.util.OperationStatus;
-import com.evolveum.midpoint.prism.PrismObject;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
 
 /**
  * Ninja action realizing "export" command.
@@ -24,7 +24,7 @@ protected String getOperationShortName() {
     }
 
     @Override
-    protected Runnable createConsumer(BlockingQueue<PrismObject<?>> queue, OperationStatus operation) {
+    protected Runnable createConsumer(BlockingQueue<ObjectType> queue, OperationStatus operation) {
         return new ExportConsumerWorker(context, options, queue, operation);
     }
 }

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/ImportRepositoryAction.java

@@ -17,7 +17,6 @@
 import com.evolveum.midpoint.ninja.opts.ImportOptions;
 import com.evolveum.midpoint.ninja.util.NinjaUtils;
 import com.evolveum.midpoint.ninja.util.OperationStatus;
-import com.evolveum.midpoint.prism.PrismObject;
 import com.evolveum.midpoint.prism.query.InOidFilter;
 import com.evolveum.midpoint.prism.query.ObjectFilter;
 import com.evolveum.midpoint.schema.result.OperationResult;
@@ -40,25 +39,25 @@ public void execute() throws Exception {
         OperationResult result = new OperationResult(OPERATION_IMPORT);
         OperationStatus progress = new OperationStatus(context, result);
 
-        BlockingQueue<PrismObject<?>> queue = new LinkedBlockingQueue<>(QUEUE_CAPACITY_PER_THREAD * options.getMultiThread());
+        BlockingQueue<ObjectType> queue = new LinkedBlockingQueue<>(QUEUE_CAPACITY_PER_THREAD * options.getMultiThread());
 
         // "+ 2" will be used for producer and progress reporter
         ExecutorService executor = Executors.newFixedThreadPool(options.getMultiThread() + 2);
 
-        ImportProducerWorker producer;
+        ImportProducerWorker<ObjectType> producer;
         if (options.getOid() != null) {
             InOidFilter filter = context.getPrismContext().queryFactory().createInOid(options.getOid());
             producer = importByFilter(filter, true, queue, progress);
         } else {
-            ObjectFilter filter = NinjaUtils.createObjectFilter(options.getFilter(), context, ObjectType.class);    // todo ok? (ObjectType)
+            ObjectFilter filter = NinjaUtils.createObjectFilter(options.getFilter(), context, ObjectType.class);
             producer = importByFilter(filter, false, queue, progress);
         }
 
         executor.execute(producer);
 
         Thread.sleep(CONSUMERS_WAIT_FOR_START);
 
-        executor.execute(new ProgressReporterWorker(context, options, queue, progress));
+        executor.execute(new ProgressReporterWorker<>(context, options, queue, progress));
 
         List<ImportRepositoryConsumerWorker> consumers = createConsumers(queue, progress);
         consumers.forEach(c -> executor.execute(c));
@@ -81,12 +80,13 @@ public LogTarget getInfoLogTarget() {
         return LogTarget.SYSTEM_ERR;
     }
 
-    private ImportProducerWorker importByFilter(ObjectFilter filter, boolean stopAfterFound,
-            BlockingQueue<PrismObject<?>> queue, OperationStatus status) {
-        return new ImportProducerWorker(context, options, queue, status, filter, stopAfterFound);
+    private ImportProducerWorker<ObjectType> importByFilter(ObjectFilter filter,
+            boolean stopAfterFound, BlockingQueue<ObjectType> queue, OperationStatus status) {
+        return new ImportProducerWorker<>(context, options, queue, status, filter, stopAfterFound);
     }
 
-    private List<ImportRepositoryConsumerWorker> createConsumers(BlockingQueue<PrismObject<?>> queue, OperationStatus operation) {
+    private List<ImportRepositoryConsumerWorker> createConsumers(
+            BlockingQueue<ObjectType> queue, OperationStatus operation) {
         List<ImportRepositoryConsumerWorker> consumers = new ArrayList<>();
 
         for (int i = 0; i < options.getMultiThread(); i++) {

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/VerifyRepositoryAction.java

@@ -11,7 +11,7 @@
 import com.evolveum.midpoint.ninja.action.worker.VerifyConsumerWorker;
 import com.evolveum.midpoint.ninja.opts.VerifyOptions;
 import com.evolveum.midpoint.ninja.util.OperationStatus;
-import com.evolveum.midpoint.prism.PrismObject;
+import com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType;
 
 /**
  * Created by Viliam Repan (lazyman).
@@ -24,7 +24,7 @@ protected String getOperationShortName() {
     }
 
     @Override
-    protected Runnable createConsumer(BlockingQueue<PrismObject<?>> queue, OperationStatus operation) {
+    protected Runnable createConsumer(BlockingQueue<ObjectType> queue, OperationStatus operation) {
         return new VerifyConsumerWorker(context, options, queue, operation);
     }
 }

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/audit/BaseAuditImportExportOptions.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2010-2021 Evolveum and contributors
+ *
+ * This work is dual-licensed under the Apache License 2.0
+ * and European Union Public License. See LICENSE file for details.
+ */
+package com.evolveum.midpoint.ninja.action.audit;
+
+import com.beust.jcommander.Parameter;
+
+import com.evolveum.midpoint.ninja.util.FileReference;
+import com.evolveum.midpoint.ninja.util.FileReferenceConverter;
+
+public class BaseAuditImportExportOptions {
+
+    public static final String P_FILTER = "-f";
+    public static final String P_FILTER_LONG = "--filter";
+
+    public static final String P_ZIP = "-z";
+    public static final String P_ZIP_LONG = "--zip";
+
+    public static final String P_MULTI_THREAD = "-l";
+    public static final String P_MULTI_THREAD_LONG = "--multi-thread";
+
+    @Parameter(names = { P_FILTER, P_FILTER_LONG }, descriptionKey = "base.filter",
+            converter = FileReferenceConverter.class, validateWith = FileReferenceConverter.class)
+    private FileReference filter;
+
+    @Parameter(names = { P_ZIP, P_ZIP_LONG }, descriptionKey = "baseImportExport.zip")
+    private boolean zip;
+
+    @Parameter(names = { P_MULTI_THREAD, P_MULTI_THREAD_LONG }, descriptionKey = "baseImportExport.multiThread")
+    private int multiThread = 1;
+
+    public FileReference getFilter() {
+        return filter;
+    }
+
+    public boolean isZip() {
+        return zip;
+    }
+
+    public int getMultiThread() {
+        return multiThread;
+    }
+}

tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/audit/ExportAuditConsumerWorker.java

@@ -12,7 +12,6 @@
 
 import com.evolveum.midpoint.ninja.action.worker.AbstractWriterConsumerWorker;
 import com.evolveum.midpoint.ninja.impl.NinjaContext;
-import com.evolveum.midpoint.ninja.opts.ExportOptions;
 import com.evolveum.midpoint.ninja.util.NinjaUtils;
 import com.evolveum.midpoint.ninja.util.OperationStatus;
 import com.evolveum.midpoint.prism.PrismSerializer;
@@ -24,12 +23,12 @@
  * Consumer writing exported audit events to the writer (stdout or file).
  */
 public class ExportAuditConsumerWorker
-        extends AbstractWriterConsumerWorker<ExportOptions, AuditEventRecordType> {
+        extends AbstractWriterConsumerWorker<ExportAuditOptions, AuditEventRecordType> {
 
     private PrismSerializer<String> serializer;
 
     public ExportAuditConsumerWorker(NinjaContext context,
-            ExportOptions options, BlockingQueue<AuditEventRecordType> queue, OperationStatus operation) {
+            ExportAuditOptions options, BlockingQueue<AuditEventRecordType> queue, OperationStatus operation) {
         super(context, options, queue, operation);
     }
 
@@ -41,7 +40,7 @@ protected void init() {
                         // TODO: This does not help with RawType: (parsed:ObjectReferenceType) for which
                         //  the names still go only to the comments (ignored by the import, obviously).
                         .serializeReferenceNames(true)
-                        .skipContainerIds(options.isSkipContainerIds()));
+                        .skipContainerIds(true));
     }
 
     @Override