From d2492d9c33f08d3313a798f8ad999f4765cb61f7 Mon Sep 17 00:00:00 2001 From: Pavol Mederly Date: Thu, 21 Sep 2017 17:30:49 +0200 Subject: [PATCH] Support for fragmented extension schemas (split into multiple files). --- .../prism/ComplexTypeDefinitionImpl.java | 12 +- .../schema/DomToSchemaPostProcessor.java | 9 +- .../prism/schema/DomToSchemaProcessor.java | 4 +- .../midpoint/prism/schema/SchemaRegistry.java | 2 +- .../prism/schema/SchemaRegistryImpl.java | 108 +++++++++++++----- .../prism/schema/XmlEntityResolverImpl.java | 18 +-- .../midpoint/prism/TestExtraSchema.java | 7 +- .../src/test/resources/schema/extension2.xsd | 41 +++++++ 8 files changed, 155 insertions(+), 46 deletions(-) create mode 100644 infra/schema/src/test/resources/schema/extension2.xsd diff --git a/infra/prism/src/main/java/com/evolveum/midpoint/prism/ComplexTypeDefinitionImpl.java b/infra/prism/src/main/java/com/evolveum/midpoint/prism/ComplexTypeDefinitionImpl.java index 36a849aadea..60b99925f14 100644 --- a/infra/prism/src/main/java/com/evolveum/midpoint/prism/ComplexTypeDefinitionImpl.java +++ b/infra/prism/src/main/java/com/evolveum/midpoint/prism/ComplexTypeDefinitionImpl.java @@ -24,6 +24,8 @@ import com.evolveum.midpoint.prism.path.ParentPathSegment; import com.evolveum.midpoint.util.DebugDumpable; import com.evolveum.midpoint.util.PrettyPrinter; +import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.util.logging.TraceManager; import org.jetbrains.annotations.NotNull; import com.evolveum.midpoint.util.QNameUtil; @@ -39,6 +41,8 @@ */ public class ComplexTypeDefinitionImpl extends TypeDefinitionImpl implements ComplexTypeDefinition { + private static final Trace LOGGER = TraceManager.getTrace(ComplexTypeDefinitionImpl.class); + private static final long serialVersionUID = 2655797837209175037L; @NotNull private final List itemDefinitions = new ArrayList<>(); private boolean containerMarker; @@ -245,7 +249,13 @@ public ID findNamedItemDefinition(@NotNull QName fir @Override public void merge(ComplexTypeDefinition otherComplexTypeDef) { for (ItemDefinition otherItemDef: otherComplexTypeDef.getDefinitions()) { - add(otherItemDef.clone()); + ItemDefinition existingItemDef = findItemDefinition(otherItemDef.getName()); + if (existingItemDef != null) { + LOGGER.warn("Overwriting existing definition {} by {} (in {})", existingItemDef, otherItemDef, this); + replaceDefinition(otherItemDef.getName(), otherItemDef.clone()); + } else { + add(otherItemDef.clone()); + } } } diff --git a/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/DomToSchemaPostProcessor.java b/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/DomToSchemaPostProcessor.java index edae5272b7a..6237d62f917 100644 --- a/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/DomToSchemaPostProcessor.java +++ b/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/DomToSchemaPostProcessor.java @@ -29,7 +29,6 @@ import org.jetbrains.annotations.NotNull; import org.w3c.dom.Element; import org.w3c.dom.NodeList; -import org.xml.sax.EntityResolver; import javax.xml.bind.annotation.XmlEnumValue; import javax.xml.namespace.QName; @@ -58,17 +57,14 @@ class DomToSchemaPostProcessor { private static final Trace LOGGER = TraceManager.getTrace(DomToSchemaPostProcessor.class); private final XSSchemaSet xsSchemaSet; - private final EntityResolver entityResolver; private final PrismContext prismContext; private PrismSchemaImpl schema; private String shortDescription; private boolean isRuntime; private boolean allowDelayedItemDefinitions; - DomToSchemaPostProcessor(XSSchemaSet xsSchemaSet, EntityResolver entityResolver, - PrismContext prismContext) { + DomToSchemaPostProcessor(XSSchemaSet xsSchemaSet, PrismContext prismContext) { this.xsSchemaSet = xsSchemaSet; - this.entityResolver = entityResolver; this.prismContext = prismContext; } @@ -126,6 +122,7 @@ private void processComplexTypeDefinitions(XSSchemaSet set) throws SchemaExcepti while (iterator.hasNext()) { XSComplexType complexType = iterator.next(); if (complexType.getTargetNamespace().equals(schema.getNamespace())) { + LOGGER.trace("### processing CTD {} into {} [{}]", complexType, schema, shortDescription); processComplexTypeDefinition(complexType); } } @@ -260,6 +257,7 @@ private void processSimpleTypeDefinitions(XSSchemaSet set) throws SchemaExceptio while (iterator.hasNext()) { XSSimpleType simpleType = iterator.next(); if (simpleType.getTargetNamespace().equals(schema.getNamespace())) { + LOGGER.trace("### processing STD {} into {} [{}]", simpleType, schema, shortDescription); processSimpleTypeDefinition(simpleType); } } @@ -532,6 +530,7 @@ private void createDefinitionsFromElements(XSSchemaSet set) throws SchemaExcepti if (xsElementDecl.getTargetNamespace().equals(schema.getNamespace())) { QName elementName = new QName(xsElementDecl.getTargetNamespace(), xsElementDecl.getName()); + LOGGER.trace("### processing item {} into {} [{}]", elementName, schema, shortDescription); XSType xsType = xsElementDecl.getType(); if (xsType == null) { throw new SchemaException("Found element " + elementName + " without type definition"); diff --git a/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/DomToSchemaProcessor.java b/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/DomToSchemaProcessor.java index 84d50ff95c9..5559223a50f 100644 --- a/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/DomToSchemaProcessor.java +++ b/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/DomToSchemaProcessor.java @@ -73,7 +73,7 @@ void parseSchema(@NotNull PrismSchemaImpl prismSchema, @NotNull Element xsdSchem if (xsSchemaSet == null) { return; } - DomToSchemaPostProcessor postProcessor = new DomToSchemaPostProcessor(xsSchemaSet, entityResolver, prismContext); + DomToSchemaPostProcessor postProcessor = new DomToSchemaPostProcessor(xsSchemaSet, prismContext); postProcessor.postprocessSchema(prismSchema, isRuntime, allowDelayedItemDefinitions, shortDescription); } @@ -89,7 +89,7 @@ void parseSchemas(List schemaDescriptions, Element wrapper, return; } for (SchemaDescription schemaDescription : schemaDescriptions) { - DomToSchemaPostProcessor postProcessor = new DomToSchemaPostProcessor(xsSchemaSet, entityResolver, prismContext); + DomToSchemaPostProcessor postProcessor = new DomToSchemaPostProcessor(xsSchemaSet, prismContext); PrismSchemaImpl prismSchema = (PrismSchemaImpl) schemaDescription.getSchema(); boolean isRuntime = schemaDescription.getCompileTimeClassesPackage() == null; String schemaShortDescription = schemaDescription.getSourceDescription() + " in " + shortDescription; diff --git a/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/SchemaRegistry.java b/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/SchemaRegistry.java index b05762732fc..d849dafdf77 100644 --- a/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/SchemaRegistry.java +++ b/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/SchemaRegistry.java @@ -48,7 +48,7 @@ public interface SchemaRegistry extends DebugDumpable, GlobalDefinitionsStore { javax.xml.validation.Schema getJavaxSchema(); - PrismSchema getSchema(String namespace); + PrismSchema getPrismSchema(String namespace); Collection getSchemas(); diff --git a/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/SchemaRegistryImpl.java b/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/SchemaRegistryImpl.java index 7bd13cbf200..892b5fd1ffe 100644 --- a/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/SchemaRegistryImpl.java +++ b/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/SchemaRegistryImpl.java @@ -45,6 +45,8 @@ import com.evolveum.midpoint.util.QNameUtil; import com.evolveum.prism.xml.ns._public.types_3.ObjectType; import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; import org.apache.commons.lang.StringUtils; import org.apache.xml.resolver.Catalog; import org.apache.xml.resolver.CatalogManager; @@ -90,7 +92,9 @@ public class SchemaRegistryImpl implements DebugDumpable, SchemaRegistry { private javax.xml.validation.Schema javaxSchema; private EntityResolver builtinSchemaResolver; final private List schemaDescriptions = new ArrayList<>(); - final private Map parsedSchemas = new HashMap<>(); + // namespace -> schemas; in case of extension schemas there could be more of them with the same namespace! + final private MultiValuedMap parsedSchemas = new ArrayListValuedHashMap<>(); + // base type name -> CTD with (merged) extension definition final private Map extensionSchemas = new HashMap<>(); private boolean initialized = false; private DynamicNamespacePrefixMapper namespacePrefixMapper; @@ -125,7 +129,7 @@ public XmlEntityResolver getEntityResolver() { return entityResolver; } - public Map getParsedSchemas() { + public MultiValuedMap getParsedSchemas() { return parsedSchemas; } @@ -359,17 +363,26 @@ public void initialize() throws SAXException, IOException, SchemaException { throw new IllegalStateException("Namespace prefix mapper not set"); } try { - LOGGER.trace("initialize() starting"); // TODO remove (all of these) + LOGGER.info("initialize() starting"); + long start = System.currentTimeMillis(); + initResolver(); - LOGGER.trace("initResolver() done"); + long resolverDone = System.currentTimeMillis(); + LOGGER.info("initResolver() done in {} ms", resolverDone - start); + parsePrismSchemas(); - LOGGER.trace("parsePrismSchemas() done"); + long prismSchemasDone = System.currentTimeMillis(); + LOGGER.info("parsePrismSchemas() done in {} ms", prismSchemasDone - resolverDone); + parseJavaxSchema(); - LOGGER.trace("parseJavaxSchema() done"); + long javaxSchemasDone = System.currentTimeMillis(); + LOGGER.info("parseJavaxSchema() done in {} ms", javaxSchemasDone - prismSchemasDone); + compileCompileTimeClassList(); - LOGGER.trace("compileCompileTimeClassList() done"); - initialized = true; + long classesDone = System.currentTimeMillis(); + LOGGER.info("compileCompileTimeClassList() done in {} ms", classesDone - javaxSchemasDone); + initialized = true; } catch (SAXException ex) { if (ex instanceof SAXParseException) { SAXParseException sex = (SAXParseException)ex; @@ -400,6 +413,16 @@ private void parsePrismSchemas() throws SchemaException { resolveMissingTypeDefinitionsInGlobalItemDefinitions((PrismSchemaImpl) schemaDescription.getSchema()); } } + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("====================================== Dumping prism schemas ======================================\n"); + for (SchemaDescription schemaDescription : schemaDescriptions) { + LOGGER.trace("************************************************************* {} (in {})", + schemaDescription.getNamespace(), schemaDescription.getPath()); + if (schemaDescription.getSchema() != null) { + LOGGER.trace("{}", schemaDescription.getSchema().debugDump()); + } + } + } } // global item definitions may refer to types that are not yet available @@ -443,35 +466,59 @@ private void parsePrismSchemas(List schemaDescriptions, boole Element schemaElement = DOMUtil.createElement(DOMUtil.XSD_SCHEMA_ELEMENT); schemaElement.setAttribute("targetNamespace", "http://dummy/"); schemaElement.setAttribute("elementFormDefault", "qualified"); + + // These fragmented namespaces should not be included in wrapper XSD because they are defined in multiple XSD files. + // We have to process them one by one. + MultiValuedMap schemasByNamespace = new ArrayListValuedHashMap<>(); + prismSchemaDescriptions.forEach(sd -> schemasByNamespace.put(sd.getNamespace(), sd)); + List fragmentedNamespaces = schemasByNamespace.keySet().stream() + .filter(ns -> schemasByNamespace.get(ns).size() > 1) + .collect(Collectors.toList()); + LOGGER.trace("Fragmented namespaces: {}", fragmentedNamespaces); + + List wrappedDescriptions = new ArrayList<>(); for (SchemaDescription description : prismSchemaDescriptions) { - Element importElement = DOMUtil.createSubElement(schemaElement, DOMUtil.XSD_IMPORT_ELEMENT); - importElement.setAttribute(DOMUtil.XSD_ATTR_NAMESPACE.getLocalPart(), description.getNamespace()); - PrismSchemaImpl schemaImpl = new PrismSchemaImpl(prismContext); - description.setSchema(schemaImpl); + String namespace = description.getNamespace(); + if (!fragmentedNamespaces.contains(namespace)) { + Element importElement = DOMUtil.createSubElement(schemaElement, DOMUtil.XSD_IMPORT_ELEMENT); + importElement.setAttribute(DOMUtil.XSD_ATTR_NAMESPACE.getLocalPart(), namespace); + description.setSchema(new PrismSchemaImpl(prismContext)); + wrappedDescriptions.add(description); + } + } + if (LOGGER.isTraceEnabled()) { + String xml = DOMUtil.serializeDOMToString(schemaElement); + LOGGER.trace("Wrapper XSD:\n{}", xml); } - //String xml = DOMUtil.serializeDOMToString(schemaElement); - //System.out.println("Wrapper XSD:\n" + xml); long started = System.currentTimeMillis(); - LOGGER.trace("Parsing {} schemas", prismSchemaDescriptions.size()); + LOGGER.trace("Parsing {} schemas wrapped in single XSD", wrappedDescriptions.size()); PrismSchemaImpl.parseSchemas(schemaElement, entityResolver, - prismSchemaDescriptions, allowDelayedItemDefinitions, getPrismContext()); + wrappedDescriptions, allowDelayedItemDefinitions, getPrismContext()); LOGGER.trace("Parsed {} schemas in {} ms", - prismSchemaDescriptions.size(), System.currentTimeMillis()-started); + wrappedDescriptions.size(), System.currentTimeMillis()-started); - for (SchemaDescription description : prismSchemaDescriptions) { + for (SchemaDescription description : wrappedDescriptions) { detectExtensionSchema(description.getSchema()); } + + for (String namespace : fragmentedNamespaces) { + Collection fragments = schemasByNamespace.get(namespace); + LOGGER.trace("Parsing {} schemas for fragmented namespace {}", fragments.size(), namespace); + for (SchemaDescription schemaDescription : fragments) { + parsePrismSchema(schemaDescription, allowDelayedItemDefinitions); + } + } } private void detectExtensionSchema(PrismSchema schema) throws SchemaException { for (ComplexTypeDefinition def: schema.getDefinitions(ComplexTypeDefinition.class)) { QName extType = def.getExtensionForType(); if (extType != null) { + LOGGER.trace("Processing {} as an extension for {}", def, extType); if (extensionSchemas.containsKey(extType)) { ComplexTypeDefinition existingExtension = extensionSchemas.get(extType); existingExtension.merge(def); -// throw new SchemaException("Duplicate definition of extension for type "+extType+": "+def+" and "+extensionSchemas.get(extType)); } else { extensionSchemas.put(extType, def.clone()); } @@ -1173,17 +1220,26 @@ private List resolveGlobalItemDefinitionsWithout //region Finding schemas @Override - public PrismSchema getSchema(String namespace) { - return parsedSchemas.get(namespace).getSchema(); + public PrismSchema getPrismSchema(String namespace) { + List schemas = parsedSchemas.get(namespace).stream() + .filter(s -> s.getSchema() != null) + .map(s -> s.getSchema()) + .collect(Collectors.toList()); + if (schemas.size() > 1) { + throw new IllegalStateException("More than one prism schema for namespace " + namespace); + } else if (schemas.size() == 1) { + return schemas.get(0); + } else { + return null; + } } @Override public Collection getSchemas() { - Collection schemas = new ArrayList(); - for (Entry entry: parsedSchemas.entrySet()) { - schemas.add(entry.getValue().getSchema()); - } - return schemas; + return parsedSchemas.values().stream() + .filter(s -> s.getSchema() != null) + .map(s -> s.getSchema()) + .collect(Collectors.toList()); } @Override diff --git a/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/XmlEntityResolverImpl.java b/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/XmlEntityResolverImpl.java index fd16d6bb10e..8de4688e81d 100644 --- a/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/XmlEntityResolverImpl.java +++ b/infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/XmlEntityResolverImpl.java @@ -25,6 +25,7 @@ import org.xml.sax.SAXException; import java.io.*; +import java.util.Collection; /** * @author semancik @@ -91,10 +92,10 @@ public LSInput resolveResource(String type, String namespaceURI, String publicId inputSource = resolveResourceUsingBuiltinResolver(type, namespaceURI, publicId, systemId, baseURI); } if (inputSource == null) { - LOGGER.error("Unable to resolve resource of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}",new Object[]{type, namespaceURI, publicId, systemId, baseURI}); + LOGGER.error("Unable to resolve resource of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}", type, namespaceURI, publicId, systemId, baseURI); return null; } - LOGGER.trace("==> Resolved resource of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {} : {}",new Object[]{type, namespaceURI, publicId, systemId, baseURI, inputSource}); + LOGGER.trace("==> Resolved resource of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {} : {}", type, namespaceURI, publicId, systemId, baseURI, inputSource); return new Input(publicId, systemId, inputSource.getByteStream()); } @@ -112,8 +113,9 @@ private InputSource resolveResourceFromRegisteredSchemas(String publicId, String private InputSource resolveResourceFromRegisteredSchemasByNamespace(String namespaceURI) { if (namespaceURI != null) { - if (schemaRegistry.getParsedSchemas().containsKey(namespaceURI)) { - SchemaDescription schemaDescription = schemaRegistry.getParsedSchemas().get(namespaceURI); + Collection schemaDescriptions = schemaRegistry.getParsedSchemas().get(namespaceURI); + if (schemaDescriptions.size() == 1) { + SchemaDescription schemaDescription = schemaDescriptions.iterator().next(); InputStream inputStream; if (schemaDescription.canInputStream()) { inputStream = schemaDescription.openInputStream(); @@ -130,6 +132,8 @@ private InputSource resolveResourceFromRegisteredSchemasByNamespace(String names source.setSystemId(namespaceURI); source.setPublicId(namespaceURI); return source; + } else { + return null; // none or ambiguous namespace } } return null; @@ -137,7 +141,7 @@ private InputSource resolveResourceFromRegisteredSchemasByNamespace(String names public InputSource resolveResourceUsingBuiltinResolver(String type, String namespaceURI, String publicId, String systemId, String baseURI) { - InputSource inputSource = null; + InputSource inputSource; try { // we first try to use traditional pair of publicId + systemId // the use of namespaceUri can be misleading in case of schema fragments: @@ -151,11 +155,11 @@ public InputSource resolveResourceUsingBuiltinResolver(String type, String names LOGGER.trace("...... Result of using builtin resolver by namespaceURI + systemId: {}", inputSource); } } catch (SAXException e) { - LOGGER.error("XML parser error resolving reference of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}: {}",new Object[]{type, namespaceURI, publicId, systemId, baseURI, e.getMessage(), e}); + LOGGER.error("XML parser error resolving reference of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}: {}", type, namespaceURI, publicId, systemId, baseURI, e.getMessage(), e); // TODO: better error handling return null; } catch (IOException e) { - LOGGER.error("IO error resolving reference of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}: {}",new Object[]{type, namespaceURI, publicId, systemId, baseURI, e.getMessage(), e}); + LOGGER.error("IO error resolving reference of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}: {}", type, namespaceURI, publicId, systemId, baseURI, e.getMessage(), e); // TODO: better error handling return null; } diff --git a/infra/prism/src/test/java/com/evolveum/midpoint/prism/TestExtraSchema.java b/infra/prism/src/test/java/com/evolveum/midpoint/prism/TestExtraSchema.java index df26a2134c6..da46857b6c2 100644 --- a/infra/prism/src/test/java/com/evolveum/midpoint/prism/TestExtraSchema.java +++ b/infra/prism/src/test/java/com/evolveum/midpoint/prism/TestExtraSchema.java @@ -41,7 +41,6 @@ import com.evolveum.midpoint.prism.foo.UserType; import com.evolveum.midpoint.prism.schema.PrismSchema; -import com.evolveum.midpoint.prism.schema.SchemaRegistry; import com.evolveum.midpoint.prism.util.PrismAsserts; import com.evolveum.midpoint.util.DOMUtil; import com.evolveum.midpoint.util.PrettyPrinter; @@ -97,13 +96,13 @@ public void testUserExtensionSchemaLoad() throws SAXException, IOException, Sche System.out.println(reg.debugDump()); // Try midpoint schemas by parsing a XML file - PrismSchema schema = reg.getSchema(NS_FOO); + PrismSchema schema = reg.getPrismSchema(NS_FOO); System.out.println("Parsed foo schema:"); System.out.println(schema.debugDump()); // TODO: assert user - schema = reg.getSchema(NS_USER_EXT); + schema = reg.getPrismSchema(NS_USER_EXT); System.out.println("Parsed user ext schema:"); System.out.println(schema.debugDump()); @@ -223,7 +222,7 @@ public void testTypeOverride() throws SAXException, IOException, SchemaException reg.registerPrismSchemasFromDirectory(EXTRA_SCHEMA_DIR); context.initialize(); - PrismSchema schema = reg.getSchema(NS_ROOT); + PrismSchema schema = reg.getPrismSchema(NS_ROOT); System.out.println("Parsed root schema:"); System.out.println(schema.debugDump()); diff --git a/infra/schema/src/test/resources/schema/extension2.xsd b/infra/schema/src/test/resources/schema/extension2.xsd new file mode 100644 index 00000000000..b489a0d8013 --- /dev/null +++ b/infra/schema/src/test/resources/schema/extension2.xsd @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + +