Permalink
Browse files

Add support for alternate schema parsers by introducing DataSchemaPar…

…ser and DataSchemaParserFactory interfaces.

RB=650205
G=si-dev
R=xma,mnchen
A=xma
1 parent 92fdc9b commit 2324d656989324463df4939c91ffa87292e2cb48 @jpbetz jpbetz committed with mchen07 Dec 2, 2015
Showing with 482 additions and 313 deletions.
  1. +3 −0 CHANGELOG
  2. +2 −1 data-avro/src/main/java/com/linkedin/data/avro/SchemaTranslator.java
  3. +4 −2 data-avro/src/test/java/com/linkedin/data/avro/TestCustomAvroSchema.java
  4. +2 −1 data-avro/src/test/java/com/linkedin/data/avro/TestSchemaTranslator.java
  5. +218 −9 data/src/main/java/com/linkedin/data/schema/{AbstractDataParser.java → AbstractSchemaParser.java}
  6. +14 −0 data/src/main/java/com/linkedin/data/schema/DataSchemaParserFactory.java
  7. +117 −0 data/src/main/java/com/linkedin/data/schema/PegasusSchemaParser.java
  8. +5 −207 data/src/main/java/com/linkedin/data/schema/SchemaParser.java
  9. +1 −1 data/src/main/java/com/linkedin/data/schema/SchemaParserFactory.java
  10. +2 −1 data/src/main/java/com/linkedin/data/schema/generator/AbstractGenerator.java
  11. +3 −2 data/src/main/java/com/linkedin/data/schema/generator/SchemaSampleDataGenerator.java
  12. +6 −5 data/src/main/java/com/linkedin/data/schema/resolver/AbstractDataSchemaResolver.java
  13. +8 −7 data/src/main/java/com/linkedin/data/schema/resolver/FileDataSchemaResolver.java
  14. +3 −2 data/src/main/java/com/linkedin/data/schema/util/Conversions.java
  15. +2 −1 data/src/main/java/com/linkedin/data/schema/util/Filters.java
  16. +2 −3 data/src/main/java/com/linkedin/data/template/DataTemplateUtil.java
  17. +7 −5 data/src/test/java/com/linkedin/data/TestUtil.java
  18. +11 −11 data/src/test/java/com/linkedin/data/schema/TestDataSchema.java
  19. +3 −2 data/src/test/java/com/linkedin/data/schema/resolver/TestDataSchemaResolver.java
  20. +4 −2 data/src/test/java/com/linkedin/data/schema/util/TestConversions.java
  21. +2 −2 data/src/test/java/com/linkedin/data/schema/validator/TestAnyRecordValidator.java
  22. +15 −7 generator/src/main/java/com/linkedin/pegasus/generator/DataSchemaParser.java
  23. +44 −38 generator/src/main/java/com/linkedin/pegasus/generator/TemplateSpecGenerator.java
  24. +3 −3 restli-docgen/src/main/java/com/linkedin/restli/docgen/RestLiResourceRelationship.java
  25. +1 −1 ...ols/src/main/java/com/linkedin/restli/tools/idlcheck/RestLiResourceModelCompatibilityChecker.java
View
@@ -9,6 +9,9 @@ Notify listeners when SimpleLoadBalancerState shutdown
(RB=648493)
Compare ReadOnly and CreateOnly paths properly in RestLiAnnotationReader.
+(RB=650205)
+Add support for alternate schema parsers by introducing PegasusSchemaParser and DataSchemaParserFactory interface.
+
5.0.7
-----
(RB=635961)
@@ -33,6 +33,7 @@
import com.linkedin.data.schema.RecordDataSchema;
import com.linkedin.data.schema.SchemaParser;
import com.linkedin.data.schema.SchemaParserFactory;
+import com.linkedin.data.schema.PegasusSchemaParser;
import com.linkedin.data.schema.UnionDataSchema;
import com.linkedin.data.schema.resolver.DefaultDataSchemaResolver;
import com.linkedin.data.schema.resolver.FileDataSchemaResolver;
@@ -125,7 +126,7 @@ public static DataSchema avroToDataSchema(String avroSchemaInJson, AvroToDataSch
SchemaParserFactory parserFactory = SchemaParserFactory.instance(validationOptions);
DataSchemaResolver resolver = getResolver(parserFactory, options);
- SchemaParser parser = parserFactory.create(resolver);
+ PegasusSchemaParser parser = parserFactory.create(resolver);
parser.parse(avroSchemaInJson);
if (parser.hasError())
{
@@ -22,8 +22,10 @@
import com.linkedin.data.avro.util.AvroUtil;
import com.linkedin.data.schema.DataSchema;
import com.linkedin.data.schema.RecordDataSchema;
-import com.linkedin.data.schema.SchemaParser;
+
import java.io.IOException;
+
+import com.linkedin.data.schema.PegasusSchemaParser;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.testng.annotations.Test;
@@ -298,7 +300,7 @@ private void translate(String dataSchemaFieldsJson, String avroSchemaFieldsJson,
.replace("##ANYRECORD_NAME", ANYRECORD_AVRO_FULL_NAME);
String fullAvroSchemaJson = AVRO_SCHEMA_JSON_TEMPLATE.replace("##FIELDS", avroSchemaFieldsJsonAfterVariableExpansion);
- SchemaParser parser = TestUtil.schemaParserFromString(fullSchemaJson);
+ PegasusSchemaParser parser = TestUtil.schemaParserFromString(fullSchemaJson);
assertFalse(parser.hasError(), parser.errorMessage());
RecordDataSchema schema = (RecordDataSchema) parser.topLevelDataSchemas().get(2);
@@ -22,6 +22,7 @@
import com.linkedin.data.schema.DataSchema;
import com.linkedin.data.schema.JsonBuilder;
import com.linkedin.data.schema.SchemaParser;
+import com.linkedin.data.schema.PegasusSchemaParser;
import com.linkedin.data.schema.SchemaToJsonEncoder;
import com.linkedin.data.schema.validation.ValidationOptions;
import java.io.BufferedReader;
@@ -1112,7 +1113,7 @@ private void testToAvroSchema(String schemaText, Object[] row) throws IOExceptio
assertTrue(schemaProperty instanceof DataMap);
// make sure embedded schema is same as the original schema
- SchemaParser schemaParser = TestUtil.schemaParserFromObjects(Arrays.asList(schemaProperty));
+ PegasusSchemaParser schemaParser = TestUtil.schemaParserFromObjects(Arrays.asList(schemaProperty));
DataSchema embeddedSchema = schemaParser.topLevelDataSchemas().get(0);
assertEquals(embeddedSchema, schema.getDereferencedDataSchema());
@@ -22,11 +22,14 @@
import com.linkedin.data.DataMap;
import com.linkedin.data.codec.DataLocation;
import com.linkedin.data.codec.JacksonDataCodec;
+import com.linkedin.data.schema.resolver.DefaultDataSchemaResolver;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -37,10 +40,209 @@
*
* @author slim
*/
-abstract public class AbstractDataParser
+abstract public class AbstractSchemaParser implements PegasusSchemaParser
{
- protected AbstractDataParser()
+ /**
+ * Constructor with resolver.
+ *
+ * @param resolver to be used to find {@link DataSchema}'s.
+ */
+ protected AbstractSchemaParser(DataSchemaResolver resolver)
+ {
+ _resolver = resolver;
+ }
+
+ /**
+ * Get the {@link DataSchemaResolver}.
+ *
+ * @return the resolver to used to find {@link DataSchema}'s, may be null
+ * if no resolver has been provided to parser.
+ */
+ public DataSchemaResolver getResolver()
+ {
+ return _resolver;
+ }
+
+ /**
+ * Return the top level {@link DataSchema}'s.
+ *
+ * The top level DataSchema's represent the types
+ * that are not defined within other types.
+ *
+ * @return the list of top level {@link DataSchema}'s in the
+ * order that are defined.
+ */
+ public List<DataSchema> topLevelDataSchemas()
+ {
+ return Collections.unmodifiableList(_topLevelDataSchemas);
+ }
+
+ public Map<Object, DataLocation> dataLocationMap()
+ {
+ return _dataLocationMap;
+ }
+
+
+ /**
+ * Bind name and aliases to {@link NamedDataSchema}.
+ *
+ * @param name to bind.
+ * @param aliasNames to bind.
+ * @param schema to be bound to the name.
+ * @return true if all names are bound to the specified {@link NamedDataSchema}.
+ */
+ protected boolean bindNameToSchema(Name name, List<Name> aliasNames, NamedDataSchema schema)
+ {
+ boolean ok = true;
+ ok &= bindNameToSchema(name, schema);
+ if (aliasNames != null)
+ {
+ for (Name aliasName : aliasNames)
+ {
+ ok &= bindNameToSchema(aliasName, schema);
+ }
+ }
+ return ok;
+ }
+
+ /**
+ * Bind a name to {@link NamedDataSchema}.
+ *
+ * @param name to bind.
+ * @param schema to be bound to the name.
+ * @return true if name is bound to the specified {@link NamedDataSchema}.
+ */
+ public boolean bindNameToSchema(Name name, NamedDataSchema schema)
+ {
+ boolean ok = true;
+ String fullName = name.getFullName();
+ if (name.isEmpty())
+ {
+ ok = false;
+ }
+ if (ok && DataSchemaUtil.typeStringToPrimitiveDataSchema(fullName) != null)
+ {
+ startErrorMessage(name).append("\"").append(fullName).append("\" is a pre-defined type and cannot be redefined.\n");
+ ok = false;
+ }
+ if (ok)
+ {
+ DataSchema found = getResolver().existingDataSchema(name.getFullName());
+ if (found != null)
+ {
+ startErrorMessage(name).append("\"").append(name.getFullName()).append("\" already defined as " + found + ".\n");
+ ok = false;
+ }
+ else
+ {
+ getResolver().bindNameToSchema(name, schema, getLocation());
+ }
+ }
+ return ok;
+ }
+
+ /**
+ * Look for {@link DataSchema} with the specified name.
+ *
+ * @param fullName to lookup.
+ * @return the {@link DataSchema} if lookup was successful else return null.
+ */
+ public DataSchema lookupName(String fullName)
{
+ DataSchema schema = DataSchemaUtil.typeStringToPrimitiveDataSchema(fullName);
+ if (schema == null)
+ {
+ schema = getResolver().findDataSchema(fullName, errorMessageBuilder());
+ }
+ return schema;
+ }
+
+ /**
+ * Lookup a name to obtain a {@link DataSchema}.
+ *
+ * The name may identify a {@link NamedDataSchema} obtained or a primitive type.
+ *
+ * @param name to lookup.
+ * @return the {@link DataSchema} of a primitive or named type
+ * if the name can be resolved, else return null.
+ */
+ protected DataSchema stringToDataSchema(String name)
+ {
+ DataSchema schema = null;
+ // Either primitive or name
+ String fullName = computeFullName(name);
+ DataSchema found = lookupName(fullName);
+ if (found == null && !name.equals(fullName))
+ {
+ found = lookupName(name);
+ }
+ if (found == null)
+ {
+ StringBuilder sb = startErrorMessage(name).append("\"").append(name).append("\"");
+ if (!name.equals(fullName))
+ {
+ sb.append(" or \"").append(fullName).append("\"");
+ }
+ sb.append(" cannot be resolved.\n");
+ }
+ else
+ {
+ schema = found;
+ }
+ return schema;
+ }
+
+ /**
+ * Compute the full name from a name.
+ *
+ * If the name identifies a primitive type, return the name.
+ * If the name is unqualified, the full name is computed by
+ * pre-pending the current namespace and "." to the input name.
+ * If the name is a full name, i.e. it contains a ".", then
+ * return the name.
+ *
+ * @param name as input to compute the full name.
+ * @return the computed full name.
+ */
+ public String computeFullName(String name)
+ {
+ String fullname;
+ DataSchema schema = DataSchemaUtil.typeStringToPrimitiveDataSchema(name);
+ if (schema != null)
+ {
+ fullname = name;
+ }
+ else if (Name.isFullName(name) || getCurrentNamespace().isEmpty())
+ {
+ fullname = name;
+ }
+ else
+ {
+ fullname = getCurrentNamespace() + "." + name;
+ }
+ return fullname;
+ }
+
+ /**
+ * Set the current namespace.
+ *
+ * Current namespace is used to compute the full name from an unqualified name.
+ *
+ * @param namespace to set as current namespace.
+ */
+ public void setCurrentNamespace(String namespace)
+ {
+ _currentNamespace = namespace;
+ }
+
+ /**
+ * Get the current namespace.
+ *
+ * @return the current namespace.
+ */
+ public String getCurrentNamespace()
+ {
+ return _currentNamespace;
}
/**
@@ -455,13 +657,6 @@ public DataSchemaLocation getLocation()
}
/**
- * Return the map of objects to their locations in the input source.
- *
- * @return the map of objects to their locations in the input source.
- */
- public abstract Map<Object, DataLocation> dataLocationMap();
-
- /**
* Add a new mapping to the map of Data object to their locations in the input source.
*
* The new mapping is added only if both arguments are not {@code null}.
@@ -559,4 +754,18 @@ protected void appendCalleeMessage(Object object)
private static final String NAMESPACE_KEY = "namespace";
private static final String SUBSTITUTE_FOR_REQUIRED_STRING = new String();
+
+
+ protected void addTopLevelSchema(DataSchema schema) {
+ _topLevelDataSchemas.add(schema);
+ }
+
+ /**
+ * Current namespace, used to determine full name from unqualified name.
+ */
+ private String _currentNamespace = "";
+
+ private final Map<Object, DataLocation> _dataLocationMap = new IdentityHashMap<Object, DataLocation>();
+ private final List<DataSchema> _topLevelDataSchemas = new ArrayList<DataSchema>();
+ private final DataSchemaResolver _resolver;
}
@@ -0,0 +1,14 @@
+package com.linkedin.data.schema;
+
+
+public interface DataSchemaParserFactory
+{
+
+ /**
+ * Create a new parser that will use the specified resolver.
+ *
+ * @param resolver to be provided to the parser.
+ * @return a new parser.
+ */
+ PegasusSchemaParser create(DataSchemaResolver resolver);
+}
Oops, something went wrong.

0 comments on commit 2324d65

Please sign in to comment.