Merge remote-tracking branch 'refs/remotes/origin/master'

Evolveum · Jan 25, 2022 · ce42d5b · ce42d5b
2 parents 4c48f56 + 1d9ae25
commit ce42d5b
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 13 deletions.
diff --git a/infra/common/src/main/java/com/evolveum/midpoint/common/validator/EventHandler.java b/infra/common/src/main/java/com/evolveum/midpoint/common/validator/EventHandler.java
@@ -54,5 +54,11 @@ public interface EventHandler<T extends Containerable> {
      *
      * @param currentResult Operation result pointing to the particular error.
      */
-    void handleGlobalError(OperationResult currentResult);
+    default void handleGlobalError(OperationResult currentResult) {
+
+    }
+
+    default void handleGlobalError(OperationResult currentResult, Exception ex) {
+        handleGlobalError(currentResult);
+    }
 }
diff --git a/infra/common/src/main/java/com/evolveum/midpoint/common/validator/LegacyValidator.java b/infra/common/src/main/java/com/evolveum/midpoint/common/validator/LegacyValidator.java
@@ -293,7 +293,7 @@ public void validate(InputStream inputStream, OperationResult validatorResult, S
             // ex.getMessage()+" on line "+stream.getLocation().getLineNumber(),ex);
             validatorResult.recordFatalError("XML parsing error: " + ex.getMessage(), ex);
             if (handler != null) {
-                handler.handleGlobalError(validatorResult);
+                handler.handleGlobalError(validatorResult, ex);
             }
             return;
         }
@@ -316,7 +316,7 @@ private EventResult readFromStreamAndValidate(XMLStreamReader stream, OperationR
         } catch (XMLStreamException ex) {
             validatorResult.recordFatalError("XML parsing error: " + ex.getMessage(), ex);
             if (handler != null) {
-                handler.handleGlobalError(validatorResult);
+                handler.handleGlobalError(validatorResult, ex);
             }
             objectResult.recordFatalError(ex);
             return EventResult.skipObject(ex.getMessage());
@@ -428,7 +428,7 @@ private EventResult validateObjectInternal(Element objectElement, OperationResul
             }
             if (handler != null) {
                 try {
-                    handler.handleGlobalError(validatorResult);
+                    handler.handleGlobalError(validatorResult, ex);
                 } catch (RuntimeException e) {
                     // Make sure that unhandled exceptions are recorded in object result before they are rethrown
                     objectResult.recordFatalError("Internal error: handleGlobalError call failed: " + e.getMessage(), e);

diff --git a/pom.xml b/pom.xml
@@ -223,7 +223,7 @@
         -->
         <h2.version>1.4.193</h2.version>
         <!-- Spring Boot parent declares most fresh versions for JDBC drivers for all our servers -->
-        <wicket.version>9.5.0</wicket.version>
+        <wicket.version>9.7.0</wicket.version>
         <wicket.chartjs.version>0.1</wicket.chartjs.version>
         <!-- Groovy 3.0.8 is also default for Spring Boot 2.5.0 -->
         <groovy.version>3.0.8</groovy.version>

diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/ImportRepositoryAction.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/ImportRepositoryAction.java
@@ -82,7 +82,7 @@ public LogTarget getInfoLogTarget() {
 
     private ImportProducerWorker<ObjectType> importByFilter(ObjectFilter filter,
             boolean stopAfterFound, BlockingQueue<ObjectType> queue, OperationStatus status) {
-        return new ImportProducerWorker<>(context, options, queue, status, filter, stopAfterFound);
+        return new ImportProducerWorker<>(context, options, queue, status, filter, stopAfterFound, options.isContinueOnInputError());
     }
 
     private List<ImportRepositoryConsumerWorker> createConsumers(

diff --git a/...a/src/main/java/com/evolveum/midpoint/ninja/action/audit/ImportAuditRepositoryAction.java b/...a/src/main/java/com/evolveum/midpoint/ninja/action/audit/ImportAuditRepositoryAction.java
@@ -76,7 +76,7 @@ public LogTarget getInfoLogTarget() {
     private ImportProducerWorker<AuditEventRecordType> importByFilter(
             ObjectFilter filter, boolean stopAfterFound,
             BlockingQueue<AuditEventRecordType> queue, OperationStatus status) {
-        return new ImportProducerWorker<>(context, options, queue, status, filter, stopAfterFound);
+        return new ImportProducerWorker<>(context, options, queue, status, filter, stopAfterFound, false);
     }
 
     private List<ImportAuditConsumerWorker> createConsumers(

diff --git a/...s/ninja/src/main/java/com/evolveum/midpoint/ninja/action/worker/ImportProducerWorker.java b/...s/ninja/src/main/java/com/evolveum/midpoint/ninja/action/worker/ImportProducerWorker.java
@@ -42,14 +42,18 @@ public class ImportProducerWorker<T extends Containerable>
 
     private final ObjectFilter filter;
     private final boolean stopAfterFound;
+    private boolean continueOnInputError;
+
+    private String currentOid = null;
 
     public ImportProducerWorker(
             NinjaContext context, BasicImportOptions options, BlockingQueue<T> queue,
-            OperationStatus operation, ObjectFilter filter, boolean stopAfterFound) {
+            OperationStatus operation, ObjectFilter filter, boolean stopAfterFound, boolean continueOnInputError) {
         super(context, options, queue, operation);
 
         this.filter = filter;
         this.stopAfterFound = stopAfterFound;
+        this.continueOnInputError = continueOnInputError;
     }
 
     @Override
@@ -119,6 +123,7 @@ private void processStream(InputStream input) throws IOException {
             @Override
             public EventResult preMarshall(Element objectElement, Node postValidationTree,
                     OperationResult objectResult) {
+                currentOid = objectElement.getAttribute("oid");
                 return EventResult.cont();
             }
 
@@ -144,22 +149,37 @@ public EventResult postMarshall(
 
                     queue.put(object);
                 } catch (Exception ex) {
-                    throw new NinjaException("Couldn't import object, reason: " + ex.getMessage(), ex);
+                    throw new NinjaException(getErrorMessage() + ", reason: " + ex.getMessage(), ex);
                 }
-
+                currentOid = null;
                 return stopAfterFound ? EventResult.skipObject() : EventResult.cont();
             }
 
             @Override
-            public void handleGlobalError(OperationResult currentResult) {
-                operation.finish();
+            public void handleGlobalError(OperationResult currentResult, Exception cause) {
+                // This should not
+                // Should we log error?
+                operation.incrementError();
+                String message = getErrorMessage();
+                if (continueOnInputError) {
+
+                    if (context.isVerbose()) {
+                        context.getLog().error(message, cause);
+                    } else {
+                        context.getLog().error(message + ", reason: {}", cause.getMessage());
+                    }
+                } else {
+                    // We need to throw runtime exception in order to stop validator, otherwise validator will continue
+                    // fill queue and this may result in deadlock
+                    operation.finish();
+                    throw new NinjaException(message + ", reason: " + cause.getMessage(), cause);
+                }
             }
         };
 
         // FIXME: MID-5151: If validateSchema is false we are not validating unknown attributes on import
         LegacyValidator<?> validator = new LegacyValidator<>(prismContext, handler);
         validator.setValidateSchema(false);
-
         OperationResult result = operation.getResult();
 
         Charset charset = context.getCharset();
@@ -180,4 +200,12 @@ private boolean matchSelectedType(Class<?> clazz) {
 
         return false;
     }
+
+    private String getErrorMessage() {
+        if (currentOid != null && !currentOid.isBlank()) {
+            return "Couldn't import object with oid '" + currentOid + "'";
+        } else {
+            return "Couldn't import object";
+        }
+    }
 }
diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/opts/ImportOptions.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/opts/ImportOptions.java
@@ -26,6 +26,9 @@ public class ImportOptions extends BaseImportExportOptions implements BasicImpor
     public static final String P_ALLOW_UNENCRYPTED_VALUES = "-e";
     public static final String P_ALLOW_UNENCRYPTED_VALUES_LONG = "--allowUnencryptedValues";
 
+    public static final String P_CONTINUE_ON_INPUT_ERROR_LONG = "--continueOnInputError";
+
+
     @Parameter(names = { P_INPUT, P_INPUT_LONG }, descriptionKey = "import.input")
     private File input;
 
@@ -36,6 +39,9 @@ public class ImportOptions extends BaseImportExportOptions implements BasicImpor
             descriptionKey = "import.allowUnencryptedValues")
     private boolean allowUnencryptedValues;
 
+    @Parameter(names = { P_CONTINUE_ON_INPUT_ERROR_LONG }, descriptionKey =  "import.continueOnInputError")
+    private boolean continueOnInputError;
+
     @Override
     public File getInput() {
         return input;
@@ -49,4 +55,10 @@ public boolean isOverwrite() {
     public boolean isAllowUnencryptedValues() {
         return allowUnencryptedValues;
     }
+
+    public boolean isContinueOnInputError() {
+        return continueOnInputError;
+    }
+
+
 }
diff --git a/tools/ninja/src/main/resources/messages.properties b/tools/ninja/src/main/resources/messages.properties
@@ -17,6 +17,7 @@ import=Imports objects into MidPoint
 import.input=Input file for import
 import.overwrite=Overwrite input file
 import.allowUnencryptedValues=Allow unencrypted values
+import.continueOnInputError=Continue with import, skipping invalid objects
 baseImportExport.raw=Use raw option
 baseImportExport.oid=Object OID
 baseImportExport.zip=Use zip/unzip compression