Permalink
Browse files

Add field aliases, field order, and include after support to PDL

  • Loading branch information...
Joe Betz
Joe Betz committed Apr 27, 2017
1 parent 6b3da4f commit fa3bafeda7b38e31c66bd9add3366416276bf9f4
View
@@ -1,6 +1,8 @@
11.0.8
------
(RB=979967)
Add field aliases, field order, and include after support to PDL.
Fix PDSC to serialize include after fields if include was after fields in original schema declaration.
11.0.7
------
@@ -75,7 +75,7 @@ propNameDeclaration returns [String name]: AT propName {
propJsonValue: EQ jsonValue;
recordDeclaration returns [String name]: RECORD identifier fieldIncludes? recordDecl=fieldSelection {
recordDeclaration returns [String name]: RECORD identifier beforeIncludes=fieldIncludes? recordDecl=fieldSelection afterIncludes=fieldIncludes? {
$name = $identifier.value;
};
@@ -569,6 +569,16 @@ public boolean isIncludeDeclaredInline(NamedDataSchema type) {
return _includesDeclaredInline.contains(type);
}
public void setFieldsBeforeIncludes(boolean fieldsBeforeIncludes)
{
_fieldsBeforeIncludes = fieldsBeforeIncludes;
}
public boolean isFieldsBeforeIncludes()
{
return _fieldsBeforeIncludes;
}
@Override
public boolean equals(Object object)
{
@@ -655,6 +665,7 @@ public int hashCode()
private Map<String, Integer> _fieldNameToIndexMap = _emptyFieldNameToIndexMap;
private final RecordType _recordType;
private Set<NamedDataSchema> _includesDeclaredInline = _emptyIncludesDeclaredInline;
private boolean _fieldsBeforeIncludes = false;
private static ThreadLocal<IdentityHashMap<RecordDataSchema, RecordDataSchema>> _equalsTracking =
new ThreadLocal<IdentityHashMap<RecordDataSchema, RecordDataSchema>>()
@@ -21,14 +21,7 @@
import com.linkedin.data.DataList;
import com.linkedin.data.DataMap;
import com.linkedin.data.codec.DataLocation;
import com.linkedin.data.message.MessageUtil;
import com.linkedin.data.schema.resolver.DefaultDataSchemaResolver;
import com.linkedin.data.schema.resolver.FileDataSchemaResolver;
import com.linkedin.data.schema.validation.CoercionMode;
import com.linkedin.data.schema.validation.RequiredMode;
import com.linkedin.data.schema.validation.ValidateDataAgainstSchema;
import com.linkedin.data.schema.validation.ValidationOptions;
import com.linkedin.data.schema.validation.ValidationResult;
import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
@@ -435,6 +428,7 @@ protected DataSchema dataMapToDataSchema(DataMap map)
// fields is before include
fields.addAll(parseFields(recordSchema, fieldsList));
fields.addAll(parseInclude(recordSchema, includeList));
recordSchema.setFieldsBeforeIncludes(true);
}
else
{
@@ -20,9 +20,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import static com.linkedin.data.schema.DataSchemaConstants.*;
@@ -258,18 +256,18 @@ protected void encodeNamed(NamedDataSchema schema, TypeRepresentation representa
break;
case RECORD:
RecordDataSchema recordDataSchema = (RecordDataSchema) schema;
if (isEncodeInclude() && recordDataSchema.getInclude().isEmpty() == false)
boolean hasIncludes = isEncodeInclude() && !recordDataSchema.getInclude().isEmpty();
boolean fieldsBeforeIncludes = recordDataSchema.isFieldsBeforeIncludes();
if (hasIncludes && !fieldsBeforeIncludes)
{
_builder.writeFieldName(INCLUDE_KEY);
_builder.writeStartArray();
for (NamedDataSchema includedSchema : recordDataSchema.getInclude())
{
encode(includedSchema);
}
_builder.writeEndArray();
writeIncludes(recordDataSchema);
}
_builder.writeFieldName(FIELDS_KEY);
encodeFields(recordDataSchema);
if (hasIncludes && fieldsBeforeIncludes)
{
writeIncludes(recordDataSchema);
}
break;
default:
throw new IllegalStateException("schema type " + schema.getType() + " is not a known NamedDataSchema type");
@@ -288,6 +286,16 @@ protected void encodeNamed(NamedDataSchema schema, TypeRepresentation representa
_currentPackage = saveCurrentPackage;
}
private void writeIncludes(RecordDataSchema recordDataSchema) throws IOException {
_builder.writeFieldName(INCLUDE_KEY);
_builder.writeStartArray();
for (NamedDataSchema includedSchema : recordDataSchema.getInclude())
{
encode(includedSchema);
}
_builder.writeEndArray();
}
protected void writeSchemaName(NamedDataSchema schema) throws IOException
{
_builder.writeString(_currentNamespace.equals(schema.getNamespace()) ? schema.getName() : schema.getFullName());
@@ -199,18 +199,9 @@ private void writeRecord(RecordDataSchema schema) throws IOException
write("record ");
write(toTypeIdentifier(schema));
List<NamedDataSchema> includes = schema.getInclude();
if (includes.size() > 0)
if (includes.size() > 0 && !schema.isFieldsBeforeIncludes())
{
write(" includes ");
for (Iterator<NamedDataSchema> iter = includes.iterator(); iter.hasNext();)
{
NamedDataSchema include = iter.next();
writeReferenceOrInline(include, schema.isIncludeDeclaredInline(include));
if (iter.hasNext())
{
write(", ");
}
}
writeIncludes(schema, includes);
}
write(" {");
newline();
@@ -227,6 +218,18 @@ private void writeRecord(RecordDataSchema schema) throws IOException
}
writeDoc(field.getDoc());
writeProperties(field.getProperties());
if (field.getOrder() != null && !field.getOrder().equals(RecordDataSchema.Field.Order.ASCENDING))
{
write("@order = \"");
write(field.getOrder().name());
write("\"");
}
if (field.getAliases() != null && field.getAliases().size() > 0)
{
write("@aliases = [");
write(field.getAliases().stream().map(s -> "\"" + s + "\"").collect(Collectors.joining(", ")));
write("]");
}
indent();
write(escapeIdentifier(field.getName()));
write(": ");
@@ -247,6 +250,24 @@ private void writeRecord(RecordDataSchema schema) throws IOException
_indentDepth--;
indent();
write("}");
if (includes.size() > 0 && schema.isFieldsBeforeIncludes())
{
writeIncludes(schema, includes);
}
}
private void writeIncludes(RecordDataSchema schema, List<NamedDataSchema> includes) throws IOException {
write(" includes ");
for (Iterator<NamedDataSchema> iter = includes.iterator(); iter.hasNext();)
{
NamedDataSchema include = iter.next();
writeReferenceOrInline(include, schema.isIncludeDeclaredInline(include));
if (iter.hasNext())
{
write(", ");
}
}
}
private void writeEnum(EnumDataSchema schema) throws IOException
@@ -519,10 +540,6 @@ private void writeProperties(List<String> prefix, Map<String, Object> properties
// Favor @x.y.z = "value" property encoding style over @x = { "y": { "z": "value" } }
writeProperties(pathParts, (DataMap) value);
}
else if (value instanceof DataList)
{
writeProperty(pathParts, CODEC.listToString((DataList) value));
}
else if (Boolean.TRUE.equals(value))
{
// Use shorthand for boolean true. Instead of writing "@deprecated = true",
@@ -25,6 +25,7 @@
import com.linkedin.data.schema.AbstractSchemaParser;
import com.linkedin.data.schema.ArrayDataSchema;
import com.linkedin.data.schema.DataSchema;
import com.linkedin.data.schema.DataSchemaConstants;
import com.linkedin.data.schema.DataSchemaResolver;
import com.linkedin.data.schema.DataSchemaUtil;
import com.linkedin.data.schema.EnumDataSchema;
@@ -624,11 +625,22 @@ private RecordDataSchema parseRecord(
RecordDataSchema schema = new RecordDataSchema(name, RecordDataSchema.RecordType.RECORD);
bindNameToSchema(name, schema);
FieldsAndIncludes fieldsAndIncludes = parseIncludes(record.fieldIncludes());
FieldsAndIncludes fieldsAndIncludes = parseIncludes(record.beforeIncludes);
boolean hasBeforeIncludes = fieldsAndIncludes.includes.size() > 0;
fieldsAndIncludes.fields.addAll(parseFields(schema, record.recordDecl));
FieldsAndIncludes afterIncludes = parseIncludes(record.afterIncludes);
boolean hasAfterIncludes = afterIncludes.includes.size() > 0;
if (hasBeforeIncludes && hasAfterIncludes)
{
startErrorMessage(record)
.append("Record may have includes before or after fields, but not both: ")
.append(record).append(NEWLINE);
}
fieldsAndIncludes.addAll(afterIncludes);
schema.setFields(fieldsAndIncludes.fields, errorMessageBuilder());
schema.setInclude(fieldsAndIncludes.includes);
schema.setIncludesDeclaredInline(fieldsAndIncludes.includesDeclaredInline);
schema.setFieldsBeforeIncludes(hasAfterIncludes);
validateDefaults(schema);
setProperties(context, schema);
return schema;
@@ -764,6 +776,13 @@ public FieldsAndIncludes(List<Field> fields, List<NamedDataSchema> includes, Set
this.includes = includes;
this.includesDeclaredInline = includesDeclaredInline;
}
public void addAll(FieldsAndIncludes includes)
{
this.fields.addAll(includes.fields);
this.includes.addAll(includes.includes);
this.includesDeclaredInline.addAll(includes.includesDeclaredInline);
}
}
private FieldsAndIncludes parseIncludes(PdlParser.FieldIncludesContext includeSet) throws ParseException
@@ -837,14 +856,76 @@ private FieldsAndIncludes parseIncludes(PdlParser.FieldIncludesContext includeSe
}
}
List<String> aliases = new ArrayList<>(0);
RecordDataSchema.Field.Order sortOrder = null;
for (PropDeclarationContext prop : field.props)
{
addPropertiesAtPath(properties, prop);
if (prop.name.equals(DataSchemaConstants.ALIASES_KEY))
{
Object value = parsePropValue(prop);
if (!(value instanceof DataList))
{
startErrorMessage(prop)
.append("'aliases' must be a list, but found ")
.append(prop.getText()).append(NEWLINE);
}
else
{
for (Object alias : (DataList) value) {
if (!(alias instanceof String))
{
startErrorMessage(prop)
.append("'aliases' list elements must be string, but found ")
.append(alias.getClass())
.append(" at ")
.append(prop.getText()).append(NEWLINE);
}
else
{
aliases.add((String) alias);
}
}
}
}
else if (prop.name.equals(DataSchemaConstants.ORDER_KEY))
{
Object value = parsePropValue(prop);
if (!(value instanceof String))
{
startErrorMessage(prop)
.append("'order' must be string, but found ")
.append(prop.getText()).append(NEWLINE);
}
else
{
String order = (String) value;
try
{
sortOrder = RecordDataSchema.Field.Order.valueOf(order.toUpperCase());
}
catch (IllegalArgumentException exc)
{
startErrorMessage(order).append("\"").append(order).append("\" is an invalid sort order.\n");
}
}
}
else
{
addPropertiesAtPath(properties, prop);
}
}
if (field.doc != null)
{
result.setDoc(field.doc.value);
}
if (aliases.size() > 0)
{
result.setAliases(aliases, errorMessageBuilder());
}
if (sortOrder != null)
{
result.setOrder(sortOrder);
}
result.setProperties(properties);
result.setRecord(recordSchema);
result.setDeclaredInline(isDeclaredInline(field.type));
@@ -560,6 +560,7 @@ public void testSchemaParser() throws IOException
},
// order of processing includes and fields is important when includes defines a named type
// include before fields
// retains order of fields and include
{
"{ " +
" \"type\" : \"record\", " +
@@ -573,8 +574,8 @@ public void testSchemaParser() throws IOException
"}",
"{ \"type\" : \"record\", \"name\" : \"Foo\", \"include\" : [ { \"type\" : \"record\", \"name\" : \"Bar\", \"fields\" : [ ] } ], \"fields\" : [ { \"name\" : \"b1\", \"type\" : \"Bar\" } ] }"
},
// order of processing includes and fields is important when includes defines a named type,
// fields before include
// retains order of fields and include
{
"{ " +
" \"type\" : \"record\", " +
@@ -586,7 +587,7 @@ public void testSchemaParser() throws IOException
" \"Bar\" " +
" ] " +
"}",
"{ \"type\" : \"record\", \"name\" : \"Foo\", \"include\" : [ { \"type\" : \"record\", \"name\" : \"Bar\", \"fields\" : [ ] } ], \"fields\" : [ { \"name\" : \"b1\", \"type\" : \"Bar\" } ] }"
"{ \"type\" : \"record\", \"name\" : \"Foo\", \"fields\" : [ { \"name\" : \"b1\", \"type\" : { \"type\" : \"record\", \"name\" : \"Bar\", \"fields\" : [ ] } } ], \"include\" : [ \"Bar\" ] }"
}
};
@@ -39,9 +39,12 @@ public void testEncode() throws IOException
assertRoundTrip("enums.DeprecatedSymbols");
assertRoundTrip("escaping.PdlKeywordEscaping");
assertRoundTrip("fixed.Fixed8");
assertRoundTrip("maps.WithOrders");
assertRoundTrip("maps.WithPrimitivesMap");
assertRoundTrip("records.Note");
assertRoundTrip("records.WithAliases");
assertRoundTrip("records.WithInclude");
assertRoundTrip("records.WithIncludeAfter");
assertRoundTrip("records.WithInlineRecord");
assertRoundTrip("records.WithPrimitives");
assertRoundTrip("records.WithOptionalPrimitives");
Oops, something went wrong.

0 comments on commit fa3bafe

Please sign in to comment.