Compat import option (+tests)

Evolveum · Nov 4, 2019 · 55492ea · 55492ea
1 parent fd21d3a
commit 55492ea
Show file tree

Hide file tree

Showing 11 changed files with 193 additions and 46 deletions.
diff --git a/infra/common/src/main/java/com/evolveum/midpoint/common/validator/LegacyValidator.java b/infra/common/src/main/java/com/evolveum/midpoint/common/validator/LegacyValidator.java
@@ -23,6 +23,7 @@
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.validation.Schema;
 
+import com.evolveum.midpoint.prism.PrismParserNoIO;
 import com.evolveum.midpoint.util.QNameUtil;
 import org.apache.commons.lang.StringUtils;
 import org.codehaus.staxmate.dom.DOMConverter;
@@ -75,6 +76,7 @@ public class LegacyValidator {
     private long progress = 0;
     private long errors = 0;
     private long stopAfterErrors = 0;
+    private boolean compatMode = false;
 
     public LegacyValidator(PrismContext prismContext) {
         this.prismContext = prismContext;
@@ -158,6 +160,14 @@ public long getErrors() {
         return errors;
     }
 
+    public boolean isCompatMode() {
+        return compatMode;
+    }
+
+    public void setCompatMode(boolean compatMode) {
+        this.compatMode = compatMode;
+    }
+
     public void validate(String lexicalRepresentation, OperationResult validationResult, String objectResultOperationName) {
         try {
             try (ByteArrayInputStream is = new ByteArrayInputStream(lexicalRepresentation.getBytes(INPUT_STREAM_CHARSET))) {
@@ -373,7 +383,11 @@ private EventResult validateObjectInternal(Element objectElement, OperationResul
                 return EventResult.skipObject();
             }
 
-            PrismObject<? extends Objectable> object = prismContext.parserFor(objectElement).parse();
+            PrismParserNoIO parser = prismContext.parserFor(objectElement);
+            if (compatMode) {
+                parser = parser.compat();
+            }
+            PrismObject<? extends Objectable> object = parser.parse();
 
             try {
                 object.checkConsistence();

diff --git a/...prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismParserImpl.java b/...prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismParserImpl.java
@@ -12,6 +12,8 @@
 import com.evolveum.midpoint.prism.impl.lex.LexicalProcessor;
 import com.evolveum.midpoint.prism.impl.xnode.RootXNodeImpl;
 import com.evolveum.midpoint.util.exception.SchemaException;
+import com.evolveum.midpoint.util.logging.Trace;
+import com.evolveum.midpoint.util.logging.TraceManager;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -26,6 +28,8 @@
  */
 abstract class PrismParserImpl implements PrismParser {
 
+    private static final Trace LOGGER = TraceManager.getTrace(PrismParserImpl.class);
+
     @NotNull private final ParserSource source;
     private final String language;
     @NotNull private final ParsingContext context;

diff --git a/...ism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismUnmarshaller.java b/...ism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismUnmarshaller.java
@@ -273,7 +273,7 @@ private <C extends Containerable> PrismContainerValue<C> parseContainerValueFrom
             ItemDefinition itemDef = locateItemDefinition(itemName, complexTypeDefinition, entry.getValue());
             if (itemDef == null) {
                 SchemaMigration migration = determineSchemaMigration(complexTypeDefinition, itemName, pc);
-                if (migration != null && !pc.isStrict()) {
+                if (migration != null && pc.isCompat()) {
                     if (migration.getOperation() == SchemaMigrationOperation.REMOVED) {
                         LOGGER.warn("Item {} was removed from the schema, skipping", itemName);
                         continue;

diff --git a/infra/schema/src/main/resources/xml/ns/public/common/api-types-3.xsd b/infra/schema/src/main/resources/xml/ns/public/common/api-types-3.xsd
@@ -173,6 +173,14 @@
                     </xsd:documentation>
                 </xsd:annotation>
             </xsd:element>
+            <xsd:element name="compatMode" type="xsd:boolean" minOccurs="0" default="false">
+                <xsd:annotation>
+                    <xsd:documentation>
+                        Compatibility model. If selected then the data parsing will be less strict.
+                        E.g. removed element will be ingnored.
+                    </xsd:documentation>
+                </xsd:annotation>
+            </xsd:element>
         </xsd:sequence>
     </xsd:complexType>
 

diff --git a/model/model-api/src/main/java/com/evolveum/midpoint/model/api/ModelService.java b/model/model-api/src/main/java/com/evolveum/midpoint/model/api/ModelService.java
@@ -609,20 +609,6 @@ void importFromResource(String shadowOid, Task task, OperationResult parentResul
      */
     void importObjectsFromFile(File input, ImportOptionsType options, Task task, OperationResult parentResult) throws FileNotFoundException;
 
-    /**
-     * Import objects from stream.
-     *
-     * Invocation of this method will happen in foreground, as the stream cannot
-     * be serialized.
-     *
-     * The results will be provided in the task.
-     *
-     * @param input
-     * @param task
-     */
-    @Deprecated
-    void importObjectsFromStream(InputStream input, ImportOptionsType options, Task task, OperationResult parentResult);
-
     /**
      * Import objects from stream.
      *
@@ -653,7 +639,7 @@ void importFromResource(String shadowOid, Task task, OperationResult parentResul
      * @return discovered connectors
      * @throws CommunicationException error communicating with the connector host
      */
-    public Set<ConnectorType> discoverConnectors(ConnectorHostType hostType, Task task, OperationResult parentResult)
+    Set<ConnectorType> discoverConnectors(ConnectorHostType hostType, Task task, OperationResult parentResult)
             throws CommunicationException, SecurityViolationException, SchemaException, ConfigurationException, ObjectNotFoundException, ExpressionEvaluationException;
 
     /**

diff --git a/...model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java b/...model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java
@@ -1525,7 +1525,7 @@ public void importObjectsFromFile(File input, ImportOptionsType options, Task ta
             throw e;
         }
         try {
-            importObjectsFromStream(fis, options, task, parentResult);
+            importObjectsFromStream(fis, PrismContext.LANG_XML, options, task, parentResult);
         } catch (RuntimeException e) {
             result.recordFatalError(e);
             throw e;
@@ -1539,11 +1539,6 @@ public void importObjectsFromFile(File input, ImportOptionsType options, Task ta
         result.computeStatus();
     }
 
-    @Override
-    public void importObjectsFromStream(InputStream input, ImportOptionsType options, Task task, OperationResult parentResult) {
-        importObjectsFromStream(input, PrismContext.LANG_XML, options, task, parentResult);
-    }
-
     @Override
     public void importObjectsFromStream(InputStream input, String language, ImportOptionsType options, Task task, OperationResult parentResult) {
         enterModelMethod();

diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/importer/ObjectImporter.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/importer/ObjectImporter.java
@@ -135,8 +135,12 @@ public boolean handleError(Throwable t) {
                     return stopAfterErrors == 0 || errors.get() < stopAfterErrors;
                 }
             };
+            PrismParser parser = prismContext.parserFor(input).language(language);
+            if (options != null && options.isCompatMode() != null && options.isCompatMode()) {
+                parser = parser.compat();
+            }
             try {
-                prismContext.parserFor(input).language(language).parseObjectsIteratively(handler);
+                parser.parseObjectsIteratively(handler);
             } catch (SchemaException|IOException e) {
                 parentResult.recordFatalError("Couldn't parse objects to be imported: " + e.getMessage(), e);
                 LoggingUtils.logUnexpectedException(LOGGER, "Couldn't parse objects to be imported", e);
@@ -173,6 +177,9 @@ public void handleGlobalError(OperationResult currentResult) {
                 }
             }
             validator.setStopAfterErrors(stopAfterErrors);
+            if (options != null && options.isCompatMode() != null && options.isCompatMode()) {
+                validator.setCompatMode(true);
+            }
             validator.validate(input, parentResult, OperationConstants.IMPORT_OBJECT);
         }
     }

diff --git a/...-intest/src/test/java/com/evolveum/midpoint/model/intest/importer/AbstractImportTest.java b/...-intest/src/test/java/com/evolveum/midpoint/model/intest/importer/AbstractImportTest.java
@@ -97,6 +97,8 @@ public abstract class AbstractImportTest extends AbstractConfiguredModelIntegrat
     protected static final String USER_HERMAN_FILE_NAME = "user-herman";
     private static final String IMPORT_REF_FILE_NAME = "import-ref";
     private static final String BAD_IMPORT_FILE_NAME = "import-bad";
+    private static final String ROLE_ONE_LEGACY_FILE_NAME = "role-one-legacy";
+    private static final String ROLE_ONE_LEGACY_OID = "0d70504c-d094-11e8-b0cc-675c492577e7";
 
     private DummyResource dummyResource;
     private DummyResourceContoller dummyResourceCtl;
@@ -149,14 +151,14 @@ public void initSystem(Task initTask, OperationResult initResult) throws Excepti
      */
     @Test
     public void test000Integrity() {
-        TestUtil.displayTestTitle(this,"test000Integrity");
+        displayTestTitle("test000Integrity");
         assertNotNull(modelService);
         assertNotNull(repositoryService);
     }
 
     @Test
     public void test001ImportConnector() throws FileNotFoundException, ObjectNotFoundException, SchemaException {
-        TestUtil.displayTestTitle(this,"test001ImportConnector");
+        displayTestTitle("test001ImportConnector");
         // GIVEN
         Task task = taskManager.createTaskInstance();
         OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test001ImportConnector");
@@ -196,7 +198,7 @@ public void test001ImportConnector() throws FileNotFoundException, ObjectNotFoun
 
     @Test
     public void test003ImportUsers() throws Exception {
-        TestUtil.displayTestTitle(this,"test003ImportUsers");
+        displayTestTitle("test003ImportUsers");
         // GIVEN
         Task task = taskManager.createTaskInstance();
         OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test003ImportUsers");
@@ -257,7 +259,7 @@ public void test003ImportUsers() throws Exception {
     // Import the same thing again. Watch how it burns :-)
     @Test
     public void test004DuplicateImportUsers() throws Exception {
-        TestUtil.displayTestTitle(this,"test004DuplicateImportUsers");
+        displayTestTitle("test004DuplicateImportUsers");
         // GIVEN
         Task task = taskManager.createTaskInstance();
         OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test004DuplicateImportUsers");
@@ -290,7 +292,7 @@ public void test004DuplicateImportUsers() throws Exception {
     // Import the same thing again, this time with overwrite option. This should go well.
     @Test
     public void test005ImportUsersWithOverwrite() throws Exception {
-        TestUtil.displayTestTitle(this,"test005ImportUsersWithOverwrite");
+        displayTestTitle("test005ImportUsersWithOverwrite");
         // GIVEN
         Task task = taskManager.createTaskInstance();
         OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test005ImportUsersWithOverwrite");
@@ -354,7 +356,7 @@ public void test005ImportUsersWithOverwrite() throws Exception {
     // Import the same thing again, with overwrite and also while keeping OIDs
     @Test
     public void test006ImportUsersWithOverwriteKeepOid() throws Exception {
-        TestUtil.displayTestTitle(this,"test006ImportUsersWithOverwriteKeepOid");
+        displayTestTitle("test006ImportUsersWithOverwriteKeepOid");
         // GIVEN
         Task task = taskManager.createTaskInstance();
         OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test005ImportUsersWithOverwrite");
@@ -416,7 +418,7 @@ public void test006ImportUsersWithOverwriteKeepOid() throws Exception {
     @Test
     public void test020ImportTask() throws Exception {
         final String TEST_NAME = "test020ImportTask";
-        TestUtil.displayTestTitle(this, TEST_NAME);
+        displayTestTitle( TEST_NAME);
         // GIVEN
         Task task = taskManager.createTaskInstance(AbstractImportTest.class.getName() + "." + TEST_NAME);
         OperationResult result = task.getResult();
@@ -461,7 +463,7 @@ public void test020ImportTask() throws Exception {
     @Test
     public void test030ImportResource() throws Exception {
         final String TEST_NAME = "test030ImportResource";
-        TestUtil.displayTestTitle(this,TEST_NAME);
+        displayTestTitle(TEST_NAME);
         // GIVEN
         Task task = taskManager.createTaskInstance(AbstractImportTest.class.getName() + "." + TEST_NAME);
         OperationResult result = task.getResult();
@@ -512,7 +514,7 @@ public void test030ImportResource() throws Exception {
     @Test
     public void test031ReimportResource() throws Exception {
         final String TEST_NAME = "test031ReimportResource";
-        TestUtil.displayTestTitle(this,TEST_NAME);
+        displayTestTitle(TEST_NAME);
         // GIVEN
         Task task = taskManager.createTaskInstance(AbstractImportTest.class.getName() + "." + TEST_NAME);
         OperationResult result = task.getResult();
@@ -567,7 +569,7 @@ public void test031ReimportResource() throws Exception {
     @Test
     public void test032ImportResourceOidAndFilter() throws Exception {
         final String TEST_NAME = "test032ImportResourceOidAndFilter";
-        TestUtil.displayTestTitle(this,TEST_NAME);
+        displayTestTitle(TEST_NAME);
         // GIVEN
         Task task = taskManager.createTaskInstance(AbstractImportTest.class.getName() + "." + TEST_NAME);
         OperationResult result = task.getResult();
@@ -621,7 +623,7 @@ public void test032ImportResourceOidAndFilter() throws Exception {
     @Test
     public void test033ImportResourceDummyRuntime() throws Exception {
         final String TEST_NAME = "test033ImportResourceDummyRuntime";
-        TestUtil.displayTestTitle(this,TEST_NAME);
+        displayTestTitle(TEST_NAME);
         // GIVEN
         Task task = taskManager.createTaskInstance(AbstractImportTest.class.getName() + "." + TEST_NAME);
         OperationResult result = task.getResult();
@@ -660,7 +662,7 @@ public void test033ImportResourceDummyRuntime() throws Exception {
     @Test
     public void test040ImportUserHermanNoEncryption() throws Exception {
         final String TEST_NAME = "test040ImportUserHermanNoEncryption";
-        TestUtil.displayTestTitle(this,TEST_NAME);
+        displayTestTitle(TEST_NAME);
         // GIVEN
 
         InternalsConfig.readEncryptionChecks = false;
@@ -707,7 +709,7 @@ public void test040ImportUserHermanNoEncryption() throws Exception {
     @Test
     public void test050ImportUserHermanOverwriteFullProcessing() throws Exception {
         final String TEST_NAME = "test050ImportUserHermanOverwriteFullProcessing";
-        TestUtil.displayTestTitle(this,TEST_NAME);
+        displayTestTitle(TEST_NAME);
         // GIVEN
 
         Task task = taskManager.createTaskInstance();
@@ -752,7 +754,7 @@ public void test050ImportUserHermanOverwriteFullProcessing() throws Exception {
     @Test
     public void test060ImportConstrainedWrongFullProcessing() throws Exception {
         final String TEST_NAME = "test060ImportConstrainedWrongFullProcessing";
-        TestUtil.displayTestTitle(this,TEST_NAME);
+        displayTestTitle(TEST_NAME);
         // GIVEN
 
         Task task = taskManager.createTaskInstance();
@@ -790,7 +792,7 @@ public void test060ImportConstrainedWrongFullProcessing() throws Exception {
     @Test
     public void test070ImportConstrainedWrong() throws Exception {
         final String TEST_NAME = "test070ImportConstrainedWrong";
-        TestUtil.displayTestTitle(this,TEST_NAME);
+        displayTestTitle(TEST_NAME);
         // GIVEN
 
         Task task = taskManager.createTaskInstance();
@@ -826,7 +828,7 @@ public void test070ImportConstrainedWrong() throws Exception {
     @Test
     public void test100GoodRefImport() throws Exception {
         final String TEST_NAME = "test100GoodRefImport";
-        TestUtil.displayTestTitle(this,TEST_NAME);
+        displayTestTitle(TEST_NAME);
         // GIVEN
         Task task = taskManager.createTaskInstance();
         OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "." +TEST_NAME);
@@ -857,11 +859,12 @@ public void test100GoodRefImport() throws Exception {
     }
 
     @Test
-    public void test200BadImport() throws FileNotFoundException, SchemaException, ObjectNotFoundException {
-        TestUtil.displayTestTitle(this,"test200BadImport");
+    public void test200BadImport() throws Exception {
+        final String TEST_NAME = "test200BadImport";
+        displayTestTitle(TEST_NAME);
         // GIVEN
         Task task = taskManager.createTaskInstance();
-        OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test001GoodImport");
+        OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "." + TEST_NAME);
         FileInputStream stream = new FileInputStream(getFile(BAD_IMPORT_FILE_NAME, false));
 
         repositoryService.deleteObject(UserType.class, USER_JACK_OID, result);
@@ -881,13 +884,60 @@ public void test200BadImport() throws FileNotFoundException, SchemaException, Ob
             AssertJUnit.fail("Jack was not imported");
         }
 
-        List<PrismObject<UserType>> users = repositoryService.searchObjects(UserType.class, null, null, result);
+        assertUsers(8);
+    }
 
-        AssertJUnit.assertNotNull(users);
-        AssertJUnit.assertEquals("Search returned unexpected results: "+users, 8, users.size());
+    /**
+     * Role ONE has elements that has been removed. Import with default options should fail.
+     */
+    @Test
+    public void test210ImportRoleOneLegacyDefault() throws Exception {
+        final String TEST_NAME = "test210ImportRoleOneLegacyDefault";
+        displayTestTitle(TEST_NAME);
+        // GIVEN
+        Task task = taskManager.createTaskInstance();
+        OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "." + TEST_NAME);
+        FileInputStream stream = new FileInputStream(getFile(ROLE_ONE_LEGACY_FILE_NAME, false));
 
+        // WHEN
+        modelService.importObjectsFromStream(stream, getLanguage(), getDefaultImportOptions(), task, result);
+
+        // THEN
+        result.computeStatus("Failed import.");
+        display("Result after bad import", result);
+        assertFailure(result);
+
+        assertNoObject(RoleType.class, ROLE_ONE_LEGACY_OID);
+
+        assertUsers(8);
     }
 
+    /**
+     * Role ONE has elements that has been removed. Import with "compat" option should succeed.
+     */
+    @Test
+    public void test212ImportRoleOneLegacyCompat() throws Exception {
+        final String TEST_NAME = "test212ImportRoleOneLegacyCompat";
+        displayTestTitle(TEST_NAME);
+        // GIVEN
+        Task task = taskManager.createTaskInstance();
+        OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "." + TEST_NAME);
+        FileInputStream stream = new FileInputStream(getFile(ROLE_ONE_LEGACY_FILE_NAME, false));
+        ImportOptionsType options = getDefaultImportOptions();
+        options.setCompatMode(true);
+
+        // WHEN
+        modelService.importObjectsFromStream(stream, getLanguage(), options, task, result);
+
+        // THEN
+        result.computeStatus("Failed import.");
+        display("Result after import", result);
+        assertSuccess(result);
+
+        assertRoleAfter(ROLE_ONE_LEGACY_OID);
+
+        assertUsers(8);
+    }
 
     private void assertDummyResource(PrismObject<ResourceType> resource, boolean fromRepo) {
         PrismContainer<Containerable> configurationPropertiesContainer = assertResource(resource, "Dummy Resource", RESOURCE_DUMMY_NAMESPACE,