Skip to content

Commit

Permalink
Support for fragmented extension schemas (split into multiple files).
Browse files Browse the repository at this point in the history
  • Loading branch information
@mederly
mederly committed Sep 21, 2017
1 parent 9d24ca7 commit d2492d9
Show file tree
Hide file tree
Showing 8 changed files with 155 additions and 46 deletions.
12 changes: 11 additions & 1 deletion infra/prism/src/main/java/com/evolveum/midpoint/prism/ComplexTypeDefinitionImpl.java
View file
Expand Up @@ -24,6 +24,8 @@
import com.evolveum.midpoint.prism.path.ParentPathSegment;
import com.evolveum.midpoint.util.DebugDumpable;
import com.evolveum.midpoint.util.PrettyPrinter;
import com.evolveum.midpoint.util.logging.Trace;
import com.evolveum.midpoint.util.logging.TraceManager;
import org.jetbrains.annotations.NotNull;
import com.evolveum.midpoint.util.QNameUtil;

Expand All @@ -39,6 +41,8 @@
*/
public class ComplexTypeDefinitionImpl extends TypeDefinitionImpl implements ComplexTypeDefinition {

private static final Trace LOGGER = TraceManager.getTrace(ComplexTypeDefinitionImpl.class);

private static final long serialVersionUID = 2655797837209175037L;
@NotNull private final List<ItemDefinition> itemDefinitions = new ArrayList<>();
private boolean containerMarker;
Expand Down Expand Up @@ -245,7 +249,13 @@ public <ID extends ItemDefinition> ID findNamedItemDefinition(@NotNull QName fir
@Override
public void merge(ComplexTypeDefinition otherComplexTypeDef) {
for (ItemDefinition otherItemDef: otherComplexTypeDef.getDefinitions()) {
add(otherItemDef.clone());
ItemDefinition existingItemDef = findItemDefinition(otherItemDef.getName());
if (existingItemDef != null) {
LOGGER.warn("Overwriting existing definition {} by {} (in {})", existingItemDef, otherItemDef, this);
replaceDefinition(otherItemDef.getName(), otherItemDef.clone());
} else {
add(otherItemDef.clone());
}
}
}

Expand Down
9 changes: 4 additions & 5 deletions infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/DomToSchemaPostProcessor.java
View file
Expand Up @@ -29,7 +29,6 @@
import org.jetbrains.annotations.NotNull;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.EntityResolver;

import javax.xml.bind.annotation.XmlEnumValue;
import javax.xml.namespace.QName;
Expand Down Expand Up @@ -58,17 +57,14 @@ class DomToSchemaPostProcessor {
private static final Trace LOGGER = TraceManager.getTrace(DomToSchemaPostProcessor.class);

private final XSSchemaSet xsSchemaSet;
private final EntityResolver entityResolver;
private final PrismContext prismContext;
private PrismSchemaImpl schema;
private String shortDescription;
private boolean isRuntime;
private boolean allowDelayedItemDefinitions;

DomToSchemaPostProcessor(XSSchemaSet xsSchemaSet, EntityResolver entityResolver,
PrismContext prismContext) {
DomToSchemaPostProcessor(XSSchemaSet xsSchemaSet, PrismContext prismContext) {
this.xsSchemaSet = xsSchemaSet;
this.entityResolver = entityResolver;
this.prismContext = prismContext;
}

Expand Down Expand Up @@ -126,6 +122,7 @@ private void processComplexTypeDefinitions(XSSchemaSet set) throws SchemaExcepti
while (iterator.hasNext()) {
XSComplexType complexType = iterator.next();
if (complexType.getTargetNamespace().equals(schema.getNamespace())) {
LOGGER.trace("### processing CTD {} into {} [{}]", complexType, schema, shortDescription);
processComplexTypeDefinition(complexType);
}
}
Expand Down Expand Up @@ -260,6 +257,7 @@ private void processSimpleTypeDefinitions(XSSchemaSet set) throws SchemaExceptio
while (iterator.hasNext()) {
XSSimpleType simpleType = iterator.next();
if (simpleType.getTargetNamespace().equals(schema.getNamespace())) {
LOGGER.trace("### processing STD {} into {} [{}]", simpleType, schema, shortDescription);
processSimpleTypeDefinition(simpleType);
}
}
Expand Down Expand Up @@ -532,6 +530,7 @@ private void createDefinitionsFromElements(XSSchemaSet set) throws SchemaExcepti
if (xsElementDecl.getTargetNamespace().equals(schema.getNamespace())) {

QName elementName = new QName(xsElementDecl.getTargetNamespace(), xsElementDecl.getName());
LOGGER.trace("### processing item {} into {} [{}]", elementName, schema, shortDescription);
XSType xsType = xsElementDecl.getType();
if (xsType == null) {
throw new SchemaException("Found element " + elementName + " without type definition");
Expand Down
4 changes: 2 additions & 2 deletions infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/DomToSchemaProcessor.java
View file
Expand Up @@ -73,7 +73,7 @@ void parseSchema(@NotNull PrismSchemaImpl prismSchema, @NotNull Element xsdSchem
if (xsSchemaSet == null) {
return;
}
DomToSchemaPostProcessor postProcessor = new DomToSchemaPostProcessor(xsSchemaSet, entityResolver, prismContext);
DomToSchemaPostProcessor postProcessor = new DomToSchemaPostProcessor(xsSchemaSet, prismContext);
postProcessor.postprocessSchema(prismSchema, isRuntime, allowDelayedItemDefinitions, shortDescription);
}

Expand All @@ -89,7 +89,7 @@ void parseSchemas(List<SchemaDescription> schemaDescriptions, Element wrapper,
return;
}
for (SchemaDescription schemaDescription : schemaDescriptions) {
DomToSchemaPostProcessor postProcessor = new DomToSchemaPostProcessor(xsSchemaSet, entityResolver, prismContext);
DomToSchemaPostProcessor postProcessor = new DomToSchemaPostProcessor(xsSchemaSet, prismContext);
PrismSchemaImpl prismSchema = (PrismSchemaImpl) schemaDescription.getSchema();
boolean isRuntime = schemaDescription.getCompileTimeClassesPackage() == null;
String schemaShortDescription = schemaDescription.getSourceDescription() + " in " + shortDescription;
Expand Down
2 changes: 1 addition & 1 deletion infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/SchemaRegistry.java
View file
Expand Up @@ -48,7 +48,7 @@ public interface SchemaRegistry extends DebugDumpable, GlobalDefinitionsStore {

javax.xml.validation.Schema getJavaxSchema();

PrismSchema getSchema(String namespace);
PrismSchema getPrismSchema(String namespace);

Collection<PrismSchema> getSchemas();

Expand Down
108 changes: 82 additions & 26 deletions infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/SchemaRegistryImpl.java
View file
Expand Up @@ -45,6 +45,8 @@
import com.evolveum.midpoint.util.QNameUtil;
import com.evolveum.prism.xml.ns._public.types_3.ObjectType;
import com.evolveum.prism.xml.ns._public.types_3.PolyStringType;
import org.apache.commons.collections4.MultiValuedMap;
import org.apache.commons.collections4.multimap.ArrayListValuedHashMap;
import org.apache.commons.lang.StringUtils;
import org.apache.xml.resolver.Catalog;
import org.apache.xml.resolver.CatalogManager;
Expand Down Expand Up @@ -90,7 +92,9 @@ public class SchemaRegistryImpl implements DebugDumpable, SchemaRegistry {
private javax.xml.validation.Schema javaxSchema;
private EntityResolver builtinSchemaResolver;
final private List<SchemaDescription> schemaDescriptions = new ArrayList<>();
final private Map<String,SchemaDescription> parsedSchemas = new HashMap<>();
// namespace -> schemas; in case of extension schemas there could be more of them with the same namespace!
final private MultiValuedMap<String,SchemaDescription> parsedSchemas = new ArrayListValuedHashMap<>();
// base type name -> CTD with (merged) extension definition
final private Map<QName,ComplexTypeDefinition> extensionSchemas = new HashMap<>();
private boolean initialized = false;
private DynamicNamespacePrefixMapper namespacePrefixMapper;
Expand Down Expand Up @@ -125,7 +129,7 @@ public XmlEntityResolver getEntityResolver() {
return entityResolver;
}

public Map<String, SchemaDescription> getParsedSchemas() {
public MultiValuedMap<String, SchemaDescription> getParsedSchemas() {
return parsedSchemas;
}

Expand Down Expand Up @@ -359,17 +363,26 @@ public void initialize() throws SAXException, IOException, SchemaException {
throw new IllegalStateException("Namespace prefix mapper not set");
}
try {
LOGGER.trace("initialize() starting"); // TODO remove (all of these)
LOGGER.info("initialize() starting");
long start = System.currentTimeMillis();

initResolver();
LOGGER.trace("initResolver() done");
long resolverDone = System.currentTimeMillis();
LOGGER.info("initResolver() done in {} ms", resolverDone - start);

parsePrismSchemas();
LOGGER.trace("parsePrismSchemas() done");
long prismSchemasDone = System.currentTimeMillis();
LOGGER.info("parsePrismSchemas() done in {} ms", prismSchemasDone - resolverDone);

parseJavaxSchema();
LOGGER.trace("parseJavaxSchema() done");
long javaxSchemasDone = System.currentTimeMillis();
LOGGER.info("parseJavaxSchema() done in {} ms", javaxSchemasDone - prismSchemasDone);

compileCompileTimeClassList();
LOGGER.trace("compileCompileTimeClassList() done");
initialized = true;
long classesDone = System.currentTimeMillis();
LOGGER.info("compileCompileTimeClassList() done in {} ms", classesDone - javaxSchemasDone);

initialized = true;
} catch (SAXException ex) {
if (ex instanceof SAXParseException) {
SAXParseException sex = (SAXParseException)ex;
Expand Down Expand Up @@ -400,6 +413,16 @@ private void parsePrismSchemas() throws SchemaException {
resolveMissingTypeDefinitionsInGlobalItemDefinitions((PrismSchemaImpl) schemaDescription.getSchema());
}
}
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("====================================== Dumping prism schemas ======================================\n");
for (SchemaDescription schemaDescription : schemaDescriptions) {
LOGGER.trace("************************************************************* {} (in {})",
schemaDescription.getNamespace(), schemaDescription.getPath());
if (schemaDescription.getSchema() != null) {
LOGGER.trace("{}", schemaDescription.getSchema().debugDump());
}
}
}
}

// global item definitions may refer to types that are not yet available
Expand Down Expand Up @@ -443,35 +466,59 @@ private void parsePrismSchemas(List<SchemaDescription> schemaDescriptions, boole
Element schemaElement = DOMUtil.createElement(DOMUtil.XSD_SCHEMA_ELEMENT);
schemaElement.setAttribute("targetNamespace", "http://dummy/");
schemaElement.setAttribute("elementFormDefault", "qualified");

// These fragmented namespaces should not be included in wrapper XSD because they are defined in multiple XSD files.
// We have to process them one by one.
MultiValuedMap<String, SchemaDescription> schemasByNamespace = new ArrayListValuedHashMap<>();
prismSchemaDescriptions.forEach(sd -> schemasByNamespace.put(sd.getNamespace(), sd));
List<String> fragmentedNamespaces = schemasByNamespace.keySet().stream()
.filter(ns -> schemasByNamespace.get(ns).size() > 1)
.collect(Collectors.toList());
LOGGER.trace("Fragmented namespaces: {}", fragmentedNamespaces);

List<SchemaDescription> wrappedDescriptions = new ArrayList<>();
for (SchemaDescription description : prismSchemaDescriptions) {
Element importElement = DOMUtil.createSubElement(schemaElement, DOMUtil.XSD_IMPORT_ELEMENT);
importElement.setAttribute(DOMUtil.XSD_ATTR_NAMESPACE.getLocalPart(), description.getNamespace());
PrismSchemaImpl schemaImpl = new PrismSchemaImpl(prismContext);
description.setSchema(schemaImpl);
String namespace = description.getNamespace();
if (!fragmentedNamespaces.contains(namespace)) {
Element importElement = DOMUtil.createSubElement(schemaElement, DOMUtil.XSD_IMPORT_ELEMENT);
importElement.setAttribute(DOMUtil.XSD_ATTR_NAMESPACE.getLocalPart(), namespace);
description.setSchema(new PrismSchemaImpl(prismContext));
wrappedDescriptions.add(description);
}
}
if (LOGGER.isTraceEnabled()) {
String xml = DOMUtil.serializeDOMToString(schemaElement);
LOGGER.trace("Wrapper XSD:\n{}", xml);
}
//String xml = DOMUtil.serializeDOMToString(schemaElement);
//System.out.println("Wrapper XSD:\n" + xml);

long started = System.currentTimeMillis();
LOGGER.trace("Parsing {} schemas", prismSchemaDescriptions.size());
LOGGER.trace("Parsing {} schemas wrapped in single XSD", wrappedDescriptions.size());
PrismSchemaImpl.parseSchemas(schemaElement, entityResolver,
prismSchemaDescriptions, allowDelayedItemDefinitions, getPrismContext());
wrappedDescriptions, allowDelayedItemDefinitions, getPrismContext());
LOGGER.trace("Parsed {} schemas in {} ms",
prismSchemaDescriptions.size(), System.currentTimeMillis()-started);
wrappedDescriptions.size(), System.currentTimeMillis()-started);

for (SchemaDescription description : prismSchemaDescriptions) {
for (SchemaDescription description : wrappedDescriptions) {
detectExtensionSchema(description.getSchema());
}

for (String namespace : fragmentedNamespaces) {
Collection<SchemaDescription> fragments = schemasByNamespace.get(namespace);
LOGGER.trace("Parsing {} schemas for fragmented namespace {}", fragments.size(), namespace);
for (SchemaDescription schemaDescription : fragments) {
parsePrismSchema(schemaDescription, allowDelayedItemDefinitions);
}
}
}

private void detectExtensionSchema(PrismSchema schema) throws SchemaException {
for (ComplexTypeDefinition def: schema.getDefinitions(ComplexTypeDefinition.class)) {
QName extType = def.getExtensionForType();
if (extType != null) {
LOGGER.trace("Processing {} as an extension for {}", def, extType);
if (extensionSchemas.containsKey(extType)) {
ComplexTypeDefinition existingExtension = extensionSchemas.get(extType);
existingExtension.merge(def);
// throw new SchemaException("Duplicate definition of extension for type "+extType+": "+def+" and "+extensionSchemas.get(extType));
} else {
extensionSchemas.put(extType, def.clone());
}
Expand Down Expand Up @@ -1173,17 +1220,26 @@ private <ID extends ItemDefinition> List<ID> resolveGlobalItemDefinitionsWithout
//region Finding schemas

@Override
public PrismSchema getSchema(String namespace) {
return parsedSchemas.get(namespace).getSchema();
public PrismSchema getPrismSchema(String namespace) {
List<PrismSchema> schemas = parsedSchemas.get(namespace).stream()
.filter(s -> s.getSchema() != null)
.map(s -> s.getSchema())
.collect(Collectors.toList());
if (schemas.size() > 1) {
throw new IllegalStateException("More than one prism schema for namespace " + namespace);
} else if (schemas.size() == 1) {
return schemas.get(0);
} else {
return null;
}
}

@Override
public Collection<PrismSchema> getSchemas() {
Collection<PrismSchema> schemas = new ArrayList<PrismSchema>();
for (Entry<String,SchemaDescription> entry: parsedSchemas.entrySet()) {
schemas.add(entry.getValue().getSchema());
}
return schemas;
return parsedSchemas.values().stream()
.filter(s -> s.getSchema() != null)
.map(s -> s.getSchema())
.collect(Collectors.toList());
}

@Override
Expand Down
18 changes: 11 additions & 7 deletions infra/prism/src/main/java/com/evolveum/midpoint/prism/schema/XmlEntityResolverImpl.java
View file
Expand Up @@ -25,6 +25,7 @@
import org.xml.sax.SAXException;

import java.io.*;
import java.util.Collection;

/**
* @author semancik
Expand Down Expand Up @@ -91,10 +92,10 @@ public LSInput resolveResource(String type, String namespaceURI, String publicId
inputSource = resolveResourceUsingBuiltinResolver(type, namespaceURI, publicId, systemId, baseURI);
}
if (inputSource == null) {
LOGGER.error("Unable to resolve resource of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}",new Object[]{type, namespaceURI, publicId, systemId, baseURI});
LOGGER.error("Unable to resolve resource of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}", type, namespaceURI, publicId, systemId, baseURI);
return null;
}
LOGGER.trace("==> Resolved resource of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {} : {}",new Object[]{type, namespaceURI, publicId, systemId, baseURI, inputSource});
LOGGER.trace("==> Resolved resource of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {} : {}", type, namespaceURI, publicId, systemId, baseURI, inputSource);
return new Input(publicId, systemId, inputSource.getByteStream());
}

Expand All @@ -112,8 +113,9 @@ private InputSource resolveResourceFromRegisteredSchemas(String publicId, String

private InputSource resolveResourceFromRegisteredSchemasByNamespace(String namespaceURI) {
if (namespaceURI != null) {
if (schemaRegistry.getParsedSchemas().containsKey(namespaceURI)) {
SchemaDescription schemaDescription = schemaRegistry.getParsedSchemas().get(namespaceURI);
Collection<SchemaDescription> schemaDescriptions = schemaRegistry.getParsedSchemas().get(namespaceURI);
if (schemaDescriptions.size() == 1) {
SchemaDescription schemaDescription = schemaDescriptions.iterator().next();
InputStream inputStream;
if (schemaDescription.canInputStream()) {
inputStream = schemaDescription.openInputStream();
Expand All @@ -130,14 +132,16 @@ private InputSource resolveResourceFromRegisteredSchemasByNamespace(String names
source.setSystemId(namespaceURI);
source.setPublicId(namespaceURI);
return source;
} else {
return null; // none or ambiguous namespace
}
}
return null;
}

public InputSource resolveResourceUsingBuiltinResolver(String type, String namespaceURI, String publicId, String systemId,
String baseURI) {
InputSource inputSource = null;
InputSource inputSource;
try {
// we first try to use traditional pair of publicId + systemId
// the use of namespaceUri can be misleading in case of schema fragments:
Expand All @@ -151,11 +155,11 @@ public InputSource resolveResourceUsingBuiltinResolver(String type, String names
LOGGER.trace("...... Result of using builtin resolver by namespaceURI + systemId: {}", inputSource);
}
} catch (SAXException e) {
LOGGER.error("XML parser error resolving reference of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}: {}",new Object[]{type, namespaceURI, publicId, systemId, baseURI, e.getMessage(), e});
LOGGER.error("XML parser error resolving reference of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}: {}", type, namespaceURI, publicId, systemId, baseURI, e.getMessage(), e);
// TODO: better error handling
return null;
} catch (IOException e) {
LOGGER.error("IO error resolving reference of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}: {}",new Object[]{type, namespaceURI, publicId, systemId, baseURI, e.getMessage(), e});
LOGGER.error("IO error resolving reference of type {}, namespaceURI: {}, publicID: {}, systemID: {}, baseURI: {}: {}", type, namespaceURI, publicId, systemId, baseURI, e.getMessage(), e);
// TODO: better error handling
return null;
}
Expand Down
7 changes: 3 additions & 4 deletions infra/prism/src/test/java/com/evolveum/midpoint/prism/TestExtraSchema.java
View file
Expand Up @@ -41,7 +41,6 @@

import com.evolveum.midpoint.prism.foo.UserType;
import com.evolveum.midpoint.prism.schema.PrismSchema;
import com.evolveum.midpoint.prism.schema.SchemaRegistry;
import com.evolveum.midpoint.prism.util.PrismAsserts;
import com.evolveum.midpoint.util.DOMUtil;
import com.evolveum.midpoint.util.PrettyPrinter;
Expand Down Expand Up @@ -97,13 +96,13 @@ public void testUserExtensionSchemaLoad() throws SAXException, IOException, Sche
System.out.println(reg.debugDump());

// Try midpoint schemas by parsing a XML file
PrismSchema schema = reg.getSchema(NS_FOO);
PrismSchema schema = reg.getPrismSchema(NS_FOO);
System.out.println("Parsed foo schema:");
System.out.println(schema.debugDump());

// TODO: assert user

schema = reg.getSchema(NS_USER_EXT);
schema = reg.getPrismSchema(NS_USER_EXT);
System.out.println("Parsed user ext schema:");
System.out.println(schema.debugDump());

Expand Down Expand Up @@ -223,7 +222,7 @@ public void testTypeOverride() throws SAXException, IOException, SchemaException
reg.registerPrismSchemasFromDirectory(EXTRA_SCHEMA_DIR);
context.initialize();

PrismSchema schema = reg.getSchema(NS_ROOT);
PrismSchema schema = reg.getPrismSchema(NS_ROOT);
System.out.println("Parsed root schema:");
System.out.println(schema.debugDump());

Expand Down

0 comments on commit d2492d9

Please sign in to comment.