diff --git a/infra/common/src/main/java/com/evolveum/midpoint/common/validator/LegacyValidator.java b/infra/common/src/main/java/com/evolveum/midpoint/common/validator/LegacyValidator.java index 6896765fa5d..e99eb7b23db 100644 --- a/infra/common/src/main/java/com/evolveum/midpoint/common/validator/LegacyValidator.java +++ b/infra/common/src/main/java/com/evolveum/midpoint/common/validator/LegacyValidator.java @@ -23,6 +23,7 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.validation.Schema; +import com.evolveum.midpoint.prism.PrismParserNoIO; import com.evolveum.midpoint.util.QNameUtil; import org.apache.commons.lang.StringUtils; import org.codehaus.staxmate.dom.DOMConverter; @@ -75,6 +76,7 @@ public class LegacyValidator { private long progress = 0; private long errors = 0; private long stopAfterErrors = 0; + private boolean compatMode = false; public LegacyValidator(PrismContext prismContext) { this.prismContext = prismContext; @@ -158,6 +160,14 @@ public long getErrors() { return errors; } + public boolean isCompatMode() { + return compatMode; + } + + public void setCompatMode(boolean compatMode) { + this.compatMode = compatMode; + } + public void validate(String lexicalRepresentation, OperationResult validationResult, String objectResultOperationName) { try { try (ByteArrayInputStream is = new ByteArrayInputStream(lexicalRepresentation.getBytes(INPUT_STREAM_CHARSET))) { @@ -373,7 +383,11 @@ private EventResult validateObjectInternal(Element objectElement, OperationResul return EventResult.skipObject(); } - PrismObject object = prismContext.parserFor(objectElement).parse(); + PrismParserNoIO parser = prismContext.parserFor(objectElement); + if (compatMode) { + parser = parser.compat(); + } + PrismObject object = parser.parse(); try { object.checkConsistence(); diff --git a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismParserImpl.java b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismParserImpl.java index 668eb16a1a4..708c49e3728 100644 --- a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismParserImpl.java +++ b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismParserImpl.java @@ -12,6 +12,8 @@ import com.evolveum.midpoint.prism.impl.lex.LexicalProcessor; import com.evolveum.midpoint.prism.impl.xnode.RootXNodeImpl; import com.evolveum.midpoint.util.exception.SchemaException; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -26,6 +28,8 @@ */ abstract class PrismParserImpl implements PrismParser { + private static final Trace LOGGER = TraceManager.getTrace(PrismParserImpl.class); + @NotNull private final ParserSource source; private final String language; @NotNull private final ParsingContext context; diff --git a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismUnmarshaller.java b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismUnmarshaller.java index 5580de3564a..a300b5fb999 100644 --- a/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismUnmarshaller.java +++ b/infra/prism-impl/src/main/java/com/evolveum/midpoint/prism/impl/marshaller/PrismUnmarshaller.java @@ -273,7 +273,7 @@ private PrismContainerValue parseContainerValueFrom ItemDefinition itemDef = locateItemDefinition(itemName, complexTypeDefinition, entry.getValue()); if (itemDef == null) { SchemaMigration migration = determineSchemaMigration(complexTypeDefinition, itemName, pc); - if (migration != null && !pc.isStrict()) { + if (migration != null && pc.isCompat()) { if (migration.getOperation() == SchemaMigrationOperation.REMOVED) { LOGGER.warn("Item {} was removed from the schema, skipping", itemName); continue; diff --git a/infra/schema/src/main/resources/xml/ns/public/common/api-types-3.xsd b/infra/schema/src/main/resources/xml/ns/public/common/api-types-3.xsd index 444851a6a1b..4ff5a4e0ed6 100644 --- a/infra/schema/src/main/resources/xml/ns/public/common/api-types-3.xsd +++ b/infra/schema/src/main/resources/xml/ns/public/common/api-types-3.xsd @@ -173,6 +173,14 @@ + + + + Compatibility model. If selected then the data parsing will be less strict. + E.g. removed element will be ingnored. + + + diff --git a/model/model-api/src/main/java/com/evolveum/midpoint/model/api/ModelService.java b/model/model-api/src/main/java/com/evolveum/midpoint/model/api/ModelService.java index 4f3a243977e..bc4f297b0aa 100644 --- a/model/model-api/src/main/java/com/evolveum/midpoint/model/api/ModelService.java +++ b/model/model-api/src/main/java/com/evolveum/midpoint/model/api/ModelService.java @@ -609,20 +609,6 @@ void importFromResource(String shadowOid, Task task, OperationResult parentResul */ void importObjectsFromFile(File input, ImportOptionsType options, Task task, OperationResult parentResult) throws FileNotFoundException; - /** - * Import objects from stream. - * - * Invocation of this method will happen in foreground, as the stream cannot - * be serialized. - * - * The results will be provided in the task. - * - * @param input - * @param task - */ - @Deprecated - void importObjectsFromStream(InputStream input, ImportOptionsType options, Task task, OperationResult parentResult); - /** * Import objects from stream. * @@ -653,7 +639,7 @@ void importFromResource(String shadowOid, Task task, OperationResult parentResul * @return discovered connectors * @throws CommunicationException error communicating with the connector host */ - public Set discoverConnectors(ConnectorHostType hostType, Task task, OperationResult parentResult) + Set discoverConnectors(ConnectorHostType hostType, Task task, OperationResult parentResult) throws CommunicationException, SecurityViolationException, SchemaException, ConfigurationException, ObjectNotFoundException, ExpressionEvaluationException; /** diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java index 7e2149f5582..239a97f2526 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/controller/ModelController.java @@ -1525,7 +1525,7 @@ public void importObjectsFromFile(File input, ImportOptionsType options, Task ta throw e; } try { - importObjectsFromStream(fis, options, task, parentResult); + importObjectsFromStream(fis, PrismContext.LANG_XML, options, task, parentResult); } catch (RuntimeException e) { result.recordFatalError(e); throw e; @@ -1539,11 +1539,6 @@ public void importObjectsFromFile(File input, ImportOptionsType options, Task ta result.computeStatus(); } - @Override - public void importObjectsFromStream(InputStream input, ImportOptionsType options, Task task, OperationResult parentResult) { - importObjectsFromStream(input, PrismContext.LANG_XML, options, task, parentResult); - } - @Override public void importObjectsFromStream(InputStream input, String language, ImportOptionsType options, Task task, OperationResult parentResult) { enterModelMethod(); diff --git a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/importer/ObjectImporter.java b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/importer/ObjectImporter.java index 9137f199e7e..4840f5e78f0 100644 --- a/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/importer/ObjectImporter.java +++ b/model/model-impl/src/main/java/com/evolveum/midpoint/model/impl/importer/ObjectImporter.java @@ -135,8 +135,12 @@ public boolean handleError(Throwable t) { return stopAfterErrors == 0 || errors.get() < stopAfterErrors; } }; + PrismParser parser = prismContext.parserFor(input).language(language); + if (options != null && options.isCompatMode() != null && options.isCompatMode()) { + parser = parser.compat(); + } try { - prismContext.parserFor(input).language(language).parseObjectsIteratively(handler); + parser.parseObjectsIteratively(handler); } catch (SchemaException|IOException e) { parentResult.recordFatalError("Couldn't parse objects to be imported: " + e.getMessage(), e); LoggingUtils.logUnexpectedException(LOGGER, "Couldn't parse objects to be imported", e); @@ -173,6 +177,9 @@ public void handleGlobalError(OperationResult currentResult) { } } validator.setStopAfterErrors(stopAfterErrors); + if (options != null && options.isCompatMode() != null && options.isCompatMode()) { + validator.setCompatMode(true); + } validator.validate(input, parentResult, OperationConstants.IMPORT_OBJECT); } } diff --git a/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/importer/AbstractImportTest.java b/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/importer/AbstractImportTest.java index a97159a06e3..d7824ffb298 100644 --- a/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/importer/AbstractImportTest.java +++ b/model/model-intest/src/test/java/com/evolveum/midpoint/model/intest/importer/AbstractImportTest.java @@ -97,6 +97,8 @@ public abstract class AbstractImportTest extends AbstractConfiguredModelIntegrat protected static final String USER_HERMAN_FILE_NAME = "user-herman"; private static final String IMPORT_REF_FILE_NAME = "import-ref"; private static final String BAD_IMPORT_FILE_NAME = "import-bad"; + private static final String ROLE_ONE_LEGACY_FILE_NAME = "role-one-legacy"; + private static final String ROLE_ONE_LEGACY_OID = "0d70504c-d094-11e8-b0cc-675c492577e7"; private DummyResource dummyResource; private DummyResourceContoller dummyResourceCtl; @@ -149,14 +151,14 @@ public void initSystem(Task initTask, OperationResult initResult) throws Excepti */ @Test public void test000Integrity() { - TestUtil.displayTestTitle(this,"test000Integrity"); + displayTestTitle("test000Integrity"); assertNotNull(modelService); assertNotNull(repositoryService); } @Test public void test001ImportConnector() throws FileNotFoundException, ObjectNotFoundException, SchemaException { - TestUtil.displayTestTitle(this,"test001ImportConnector"); + displayTestTitle("test001ImportConnector"); // GIVEN Task task = taskManager.createTaskInstance(); OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test001ImportConnector"); @@ -196,7 +198,7 @@ public void test001ImportConnector() throws FileNotFoundException, ObjectNotFoun @Test public void test003ImportUsers() throws Exception { - TestUtil.displayTestTitle(this,"test003ImportUsers"); + displayTestTitle("test003ImportUsers"); // GIVEN Task task = taskManager.createTaskInstance(); OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test003ImportUsers"); @@ -257,7 +259,7 @@ public void test003ImportUsers() throws Exception { // Import the same thing again. Watch how it burns :-) @Test public void test004DuplicateImportUsers() throws Exception { - TestUtil.displayTestTitle(this,"test004DuplicateImportUsers"); + displayTestTitle("test004DuplicateImportUsers"); // GIVEN Task task = taskManager.createTaskInstance(); OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test004DuplicateImportUsers"); @@ -290,7 +292,7 @@ public void test004DuplicateImportUsers() throws Exception { // Import the same thing again, this time with overwrite option. This should go well. @Test public void test005ImportUsersWithOverwrite() throws Exception { - TestUtil.displayTestTitle(this,"test005ImportUsersWithOverwrite"); + displayTestTitle("test005ImportUsersWithOverwrite"); // GIVEN Task task = taskManager.createTaskInstance(); OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test005ImportUsersWithOverwrite"); @@ -354,7 +356,7 @@ public void test005ImportUsersWithOverwrite() throws Exception { // Import the same thing again, with overwrite and also while keeping OIDs @Test public void test006ImportUsersWithOverwriteKeepOid() throws Exception { - TestUtil.displayTestTitle(this,"test006ImportUsersWithOverwriteKeepOid"); + displayTestTitle("test006ImportUsersWithOverwriteKeepOid"); // GIVEN Task task = taskManager.createTaskInstance(); OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test005ImportUsersWithOverwrite"); @@ -416,7 +418,7 @@ public void test006ImportUsersWithOverwriteKeepOid() throws Exception { @Test public void test020ImportTask() throws Exception { final String TEST_NAME = "test020ImportTask"; - TestUtil.displayTestTitle(this, TEST_NAME); + displayTestTitle( TEST_NAME); // GIVEN Task task = taskManager.createTaskInstance(AbstractImportTest.class.getName() + "." + TEST_NAME); OperationResult result = task.getResult(); @@ -461,7 +463,7 @@ public void test020ImportTask() throws Exception { @Test public void test030ImportResource() throws Exception { final String TEST_NAME = "test030ImportResource"; - TestUtil.displayTestTitle(this,TEST_NAME); + displayTestTitle(TEST_NAME); // GIVEN Task task = taskManager.createTaskInstance(AbstractImportTest.class.getName() + "." + TEST_NAME); OperationResult result = task.getResult(); @@ -512,7 +514,7 @@ public void test030ImportResource() throws Exception { @Test public void test031ReimportResource() throws Exception { final String TEST_NAME = "test031ReimportResource"; - TestUtil.displayTestTitle(this,TEST_NAME); + displayTestTitle(TEST_NAME); // GIVEN Task task = taskManager.createTaskInstance(AbstractImportTest.class.getName() + "." + TEST_NAME); OperationResult result = task.getResult(); @@ -567,7 +569,7 @@ public void test031ReimportResource() throws Exception { @Test public void test032ImportResourceOidAndFilter() throws Exception { final String TEST_NAME = "test032ImportResourceOidAndFilter"; - TestUtil.displayTestTitle(this,TEST_NAME); + displayTestTitle(TEST_NAME); // GIVEN Task task = taskManager.createTaskInstance(AbstractImportTest.class.getName() + "." + TEST_NAME); OperationResult result = task.getResult(); @@ -621,7 +623,7 @@ public void test032ImportResourceOidAndFilter() throws Exception { @Test public void test033ImportResourceDummyRuntime() throws Exception { final String TEST_NAME = "test033ImportResourceDummyRuntime"; - TestUtil.displayTestTitle(this,TEST_NAME); + displayTestTitle(TEST_NAME); // GIVEN Task task = taskManager.createTaskInstance(AbstractImportTest.class.getName() + "." + TEST_NAME); OperationResult result = task.getResult(); @@ -660,7 +662,7 @@ public void test033ImportResourceDummyRuntime() throws Exception { @Test public void test040ImportUserHermanNoEncryption() throws Exception { final String TEST_NAME = "test040ImportUserHermanNoEncryption"; - TestUtil.displayTestTitle(this,TEST_NAME); + displayTestTitle(TEST_NAME); // GIVEN InternalsConfig.readEncryptionChecks = false; @@ -707,7 +709,7 @@ public void test040ImportUserHermanNoEncryption() throws Exception { @Test public void test050ImportUserHermanOverwriteFullProcessing() throws Exception { final String TEST_NAME = "test050ImportUserHermanOverwriteFullProcessing"; - TestUtil.displayTestTitle(this,TEST_NAME); + displayTestTitle(TEST_NAME); // GIVEN Task task = taskManager.createTaskInstance(); @@ -752,7 +754,7 @@ public void test050ImportUserHermanOverwriteFullProcessing() throws Exception { @Test public void test060ImportConstrainedWrongFullProcessing() throws Exception { final String TEST_NAME = "test060ImportConstrainedWrongFullProcessing"; - TestUtil.displayTestTitle(this,TEST_NAME); + displayTestTitle(TEST_NAME); // GIVEN Task task = taskManager.createTaskInstance(); @@ -790,7 +792,7 @@ public void test060ImportConstrainedWrongFullProcessing() throws Exception { @Test public void test070ImportConstrainedWrong() throws Exception { final String TEST_NAME = "test070ImportConstrainedWrong"; - TestUtil.displayTestTitle(this,TEST_NAME); + displayTestTitle(TEST_NAME); // GIVEN Task task = taskManager.createTaskInstance(); @@ -826,7 +828,7 @@ public void test070ImportConstrainedWrong() throws Exception { @Test public void test100GoodRefImport() throws Exception { final String TEST_NAME = "test100GoodRefImport"; - TestUtil.displayTestTitle(this,TEST_NAME); + displayTestTitle(TEST_NAME); // GIVEN Task task = taskManager.createTaskInstance(); OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "." +TEST_NAME); @@ -857,11 +859,12 @@ public void test100GoodRefImport() throws Exception { } @Test - public void test200BadImport() throws FileNotFoundException, SchemaException, ObjectNotFoundException { - TestUtil.displayTestTitle(this,"test200BadImport"); + public void test200BadImport() throws Exception { + final String TEST_NAME = "test200BadImport"; + displayTestTitle(TEST_NAME); // GIVEN Task task = taskManager.createTaskInstance(); - OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "test001GoodImport"); + OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "." + TEST_NAME); FileInputStream stream = new FileInputStream(getFile(BAD_IMPORT_FILE_NAME, false)); repositoryService.deleteObject(UserType.class, USER_JACK_OID, result); @@ -881,13 +884,60 @@ public void test200BadImport() throws FileNotFoundException, SchemaException, Ob AssertJUnit.fail("Jack was not imported"); } - List> users = repositoryService.searchObjects(UserType.class, null, null, result); + assertUsers(8); + } - AssertJUnit.assertNotNull(users); - AssertJUnit.assertEquals("Search returned unexpected results: "+users, 8, users.size()); + /** + * Role ONE has elements that has been removed. Import with default options should fail. + */ + @Test + public void test210ImportRoleOneLegacyDefault() throws Exception { + final String TEST_NAME = "test210ImportRoleOneLegacyDefault"; + displayTestTitle(TEST_NAME); + // GIVEN + Task task = taskManager.createTaskInstance(); + OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "." + TEST_NAME); + FileInputStream stream = new FileInputStream(getFile(ROLE_ONE_LEGACY_FILE_NAME, false)); + // WHEN + modelService.importObjectsFromStream(stream, getLanguage(), getDefaultImportOptions(), task, result); + + // THEN + result.computeStatus("Failed import."); + display("Result after bad import", result); + assertFailure(result); + + assertNoObject(RoleType.class, ROLE_ONE_LEGACY_OID); + + assertUsers(8); } + /** + * Role ONE has elements that has been removed. Import with "compat" option should succeed. + */ + @Test + public void test212ImportRoleOneLegacyCompat() throws Exception { + final String TEST_NAME = "test212ImportRoleOneLegacyCompat"; + displayTestTitle(TEST_NAME); + // GIVEN + Task task = taskManager.createTaskInstance(); + OperationResult result = new OperationResult(AbstractImportTest.class.getName() + "." + TEST_NAME); + FileInputStream stream = new FileInputStream(getFile(ROLE_ONE_LEGACY_FILE_NAME, false)); + ImportOptionsType options = getDefaultImportOptions(); + options.setCompatMode(true); + + // WHEN + modelService.importObjectsFromStream(stream, getLanguage(), options, task, result); + + // THEN + result.computeStatus("Failed import."); + display("Result after import", result); + assertSuccess(result); + + assertRoleAfter(ROLE_ONE_LEGACY_OID); + + assertUsers(8); + } private void assertDummyResource(PrismObject resource, boolean fromRepo) { PrismContainer configurationPropertiesContainer = assertResource(resource, "Dummy Resource", RESOURCE_DUMMY_NAMESPACE, diff --git a/model/model-intest/src/test/resources/importer/role-one-legacy.json b/model/model-intest/src/test/resources/importer/role-one-legacy.json new file mode 100644 index 00000000000..0765005af7f --- /dev/null +++ b/model/model-intest/src/test/resources/importer/role-one-legacy.json @@ -0,0 +1,27 @@ +{ + "role": { + "oid": "0d70504c-d094-11e8-b0cc-675c492577e7", + "name": "One", + "description": "First role", + "roleType": "lab", + "approverExpression": { + "script": { + "code": "midpoint.oid2ort(user.getOid())" + } + }, + "policyConstraints": { + "minAssignees": [ + { + "enforcement": "report", + "multiplicity": 2 + }, + { + "multiplicity": 1 + } + ], + "maxAssignees": { + "multiplicity": "unbounded" + } + } + } +} diff --git a/model/model-intest/src/test/resources/importer/role-one-legacy.xml b/model/model-intest/src/test/resources/importer/role-one-legacy.xml new file mode 100644 index 00000000000..70f4b098a86 --- /dev/null +++ b/model/model-intest/src/test/resources/importer/role-one-legacy.xml @@ -0,0 +1,38 @@ + + + + + + + One + First role + + lab + + + + + + + + report + 2 + + + 1 + + + unbounded + + + + diff --git a/model/model-intest/src/test/resources/importer/role-one-legacy.yaml b/model/model-intest/src/test/resources/importer/role-one-legacy.yaml new file mode 100644 index 00000000000..4773aa5dd28 --- /dev/null +++ b/model/model-intest/src/test/resources/importer/role-one-legacy.yaml @@ -0,0 +1,18 @@ +--- +role: + oid: 0d70504c-d094-11e8-b0cc-675c492577e7 + name: One + description: First role + roleType: lab + approverExpression: + script: + code: midpoint.oid2ort(user.getOid()) + policyConstraints: + minAssignees: + - + enforcement: report + multiplicity: 2 + - + multiplicity: 1 + maxAssignees: + multiplicity: unbounded