To enable reserialization of .pdsc and .pdl files back to why they we…

…re structured when originally parsed, add 'isInline' to all type references in DataSchema classes.

Add encoder to write DataSchemas back to .pdl source code.

RB=918607
R=xma,mnchen
A=xma

CHANGELOG

@@ -6,6 +6,9 @@ Ensuring cloned DataLists and DataMaps have unique __dataComplexHashCode values.
 (RB=922604)
 Update documentation on CreateKVResponse and BatchCreateKVResult.
 
+(RB=918607)
+Add .pdsc to .pdl conversion support
+
 10.1.4
 ------
 (RB=876655)

data-avro/src/main/java/com/linkedin/data/avro/SchemaToAvroJsonEncoder.java

@@ -108,6 +108,24 @@ public void encode(DataSchema schema) throws IOException
     }
   }
 
+  /**
+   * Encode a {@link DataSchema}.
+   *
+   * Special handling is required for typeref's. All typeref's are
+   * de-referenced to the actual type.
+   *
+   * @param schema to encode.
+   * @throws IOException
+   */
+  @Override
+  protected void encode(DataSchema schema, boolean originallyInlined) throws IOException
+  {
+    if (encodeCustomAvroSchema(schema) == false)
+    {
+      super.encode(schema.getDereferencedDataSchema(), originallyInlined);
+    }
+  }
+
   @Override
   protected void encodeProperties(DataSchema schema) throws IOException
   {

data/src/main/antlr/com/linkedin/data/grammar/Pdl.g4

@@ -47,6 +47,11 @@ typeReference returns [String value]: qualifiedIdentifier {
 
 typeDeclaration: namedTypeDeclaration | anonymousTypeDeclaration;
 
+
+// Why are named declarations are identified with qualifiedIdentifier?
+// Begrudgingly, for compatibility with .pdsc. In .pdsc all type declarations may specify a namespace,
+// and there is not restriction on what the namespace is.
+//
 // Only named declarations support schemadoc and properties.
 namedTypeDeclaration: doc=schemadoc? props+=propDeclaration*
   (recordDeclaration | enumDeclaration | typerefDeclaration | fixedDeclaration);
@@ -69,12 +74,12 @@ propNameDeclaration returns [String name]: AT qualifiedIdentifier {
 
 propJsonValue: EQ jsonValue;
 
-recordDeclaration returns [String name]: RECORD identifier fieldIncludes? recordDecl=fieldSelection {
-  $name = $identifier.value;
+recordDeclaration returns [String name]: RECORD qualifiedIdentifier fieldIncludes? recordDecl=fieldSelection {
+  $name = $qualifiedIdentifier.value;
 };
 
-enumDeclaration returns [String name]: ENUM identifier enumDecl=enumSymbolDeclarations {
-  $name = $identifier.value;
+enumDeclaration returns [String name]: ENUM qualifiedIdentifier enumDecl=enumSymbolDeclarations {
+  $name = $qualifiedIdentifier.value;
 };
 
 enumSymbolDeclarations: OPEN_BRACE symbolDecls+=enumSymbolDeclaration* CLOSE_BRACE;
@@ -85,13 +90,13 @@ enumSymbol returns [String value]: identifier {
   $value = $identifier.value;
 };
 
-typerefDeclaration returns [String name]: TYPEREF identifier EQ ref=typeAssignment {
-  $name = $identifier.value;
+typerefDeclaration returns [String name]: TYPEREF qualifiedIdentifier EQ ref=typeAssignment {
+  $name = $qualifiedIdentifier.value;
 };
 
 fixedDeclaration returns[String name, int size]:
-  FIXED identifier sizeStr=NUMBER_LITERAL {
-  $name = $identifier.value;
+  FIXED qualifiedIdentifier sizeStr=NUMBER_LITERAL {
+  $name = $qualifiedIdentifier.value;
   $size = $sizeStr.int;
 };
 
@@ -111,7 +116,7 @@ mapTypeAssignments: OPEN_BRACKET key=typeAssignment value=typeAssignment CLOSE_B
 
 fieldSelection: OPEN_BRACE fields+=fieldDeclaration* CLOSE_BRACE;
 
-fieldIncludes: INCLUDES typeReference*;
+fieldIncludes: INCLUDES typeAssignment*;
 
 fieldDeclaration returns [String name, boolean isOptional]:
     doc=schemadoc? props+=propDeclaration* fieldName=identifier COLON type=typeAssignment QUESTION_MARK?

data/src/main/java/com/linkedin/data/schema/AbstractSchemaEncoder.java

@@ -0,0 +1,140 @@
+/*
+   Copyright (c) 2012 LinkedIn Corp.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package com.linkedin.data.schema;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+
+public abstract class AbstractSchemaEncoder {
+  protected TypeReferenceFormat _typeReferenceFormat = TypeReferenceFormat.DENORMALIZE;
+  private final Set<String> _alreadyEncountered = new HashSet<>();
+
+  public AbstractSchemaEncoder()
+  {
+  }
+
+  public AbstractSchemaEncoder(TypeReferenceFormat typeReferenceFormat)
+  {
+    _typeReferenceFormat = typeReferenceFormat;
+  }
+
+  /**
+   * Encode the specified {@link DataSchema}.
+   * @param schema to encode.
+   * @throws IOException if there is an error while encoding.
+   */
+  abstract public void encode(DataSchema schema) throws IOException;
+
+  /**
+   * The different ways type references can be formatted.
+   */
+  public enum TypeReferenceFormat
+  {
+    /**
+     * Format with all dependent types declared inline at their first lexical appearance, and referenced by name in
+     * all subsequent appearances.
+     *
+     * This format produces a single JSON object representation of this schema with all of the schemas it
+     * transitively depends on inlined.
+     */
+    DENORMALIZE,
+
+    /**
+     * Format with all dependent types either declared inline or referenced in the exact same way they were in the
+     * original schema declaration.
+     */
+    PRESERVE,
+
+    /**
+     * Format with all dependent types referenced by name.
+     */
+    MINIMIZE
+  }
+
+  /**
+   * Gets how type references are formatted.
+   */
+  public TypeReferenceFormat getTypeReferenceFormat()
+  {
+    return _typeReferenceFormat;
+  }
+
+  /**
+   * Set how type references are formatted.
+   */
+  public void setTypeReferenceFormat(TypeReferenceFormat typeReferenceFormat)
+  {
+    _typeReferenceFormat = typeReferenceFormat;
+  }
+
+  /**
+   * Determines how a type from the original schema should be encoded.
+   *
+   * @param originallyInlined identifies if the provided type was originally inlined.
+   * @return the {@link TypeRepresentation} to use when encoding a type.
+   */
+  protected TypeRepresentation selectTypeRepresentation(DataSchema schema, boolean originallyInlined)
+  {
+    boolean firstEncounter = true;
+    if (schema instanceof NamedDataSchema)
+    {
+      String fullName = ((NamedDataSchema) schema).getFullName();
+      firstEncounter = !_alreadyEncountered.contains(fullName);
+    }
+    else if (schema instanceof PrimitiveDataSchema)
+    {
+      return TypeRepresentation.DECLARED_INLINE;
+    }
+    switch (_typeReferenceFormat)
+    {
+      case PRESERVE:
+        return originallyInlined ? TypeRepresentation.DECLARED_INLINE : TypeRepresentation.REFERENCED_BY_NAME;
+      case DENORMALIZE:
+        return firstEncounter ? TypeRepresentation.DECLARED_INLINE : TypeRepresentation.REFERENCED_BY_NAME;
+      case MINIMIZE:
+        return TypeRepresentation.REFERENCED_BY_NAME;
+      default:
+        throw new IllegalArgumentException("Unrecognized enum symbol: " + _typeReferenceFormat);
+    }
+  }
+
+  public void markEncountered(DataSchema schema)
+  {
+    if (schema instanceof NamedDataSchema)
+    {
+      _alreadyEncountered.add(((NamedDataSchema) schema).getFullName());
+    }
+  }
+
+  /**
+   * Possible serialization formats of a particular dependant type.
+   */
+  protected enum TypeRepresentation
+  {
+    /**
+     * The type declaration is inlined.
+     */
+    DECLARED_INLINE,
+
+    /**
+     * The type is referenced by name.
+     */
+    REFERENCED_BY_NAME
+  }
+}

data/src/main/java/com/linkedin/data/schema/ArrayDataSchema.java

@@ -57,6 +57,24 @@ public DataSchema getItems()
     return _items;
   }
 
+  /**
+   * Sets if the items type is declared inline in the schema.
+   * @param itemsDeclaredInline true if the items type is declared inline, false if it is referenced by name.
+   */
+  public void setItemsDeclaredInline(boolean itemsDeclaredInline)
+  {
+    _itemsDeclaredInline = itemsDeclaredInline;
+  }
+
+  /**
+   * Checks if the item type is declared inline.
+   * @return true if the items type is declared inline, false if it is referenced by name.
+   */
+  public boolean isItemsDeclaredInline()
+  {
+    return _itemsDeclaredInline;
+  }
+
   @Override
   public String getUnionMemberKey()
   {
@@ -85,4 +103,5 @@ public int hashCode()
   }
 
   private DataSchema _items = DataSchemaConstants.NULL_DATA_SCHEMA;
+  private boolean _itemsDeclaredInline = false;
 }

data/src/main/java/com/linkedin/data/schema/MapDataSchema.java

@@ -57,6 +57,24 @@ public DataSchema getValues()
     return _values;
   }
 
+  /**
+   * Sets if the values type is declared inline in the schema.
+   * @param valuesDeclaredInline true if the values type is declared inline, false if it is referenced by name.
+   */
+  public void setValuesDeclaredInline(boolean valuesDeclaredInline)
+  {
+    _valuesDeclaredInline = valuesDeclaredInline;
+  }
+
+  /**
+   * Checks if the values type is declared inline.
+   * @return true if the values type is declared inline, false if it is referenced by name.
+   */
+  public boolean isValuesDeclaredInline()
+  {
+    return _valuesDeclaredInline;
+  }
+
   @Override
   public String getUnionMemberKey()
   {
@@ -85,4 +103,5 @@ public int hashCode()
   }
 
   private DataSchema _values = DataSchemaConstants.NULL_DATA_SCHEMA;
+  private boolean _valuesDeclaredInline = false;
 }

data/src/main/java/com/linkedin/data/schema/Name.java

@@ -21,7 +21,7 @@
 import static com.linkedin.data.schema.DataSchemaConstants.NAME_PATTERN;
 import static com.linkedin.data.schema.DataSchemaConstants.UNQUALIFIED_NAME_PATTERN;
 
-public final class Name
+public final class Name implements Comparable<Name>
 {
   /**
    * Construct empty {@link Name}.
@@ -51,7 +51,7 @@ public Name(String fullName)
    * append errors in the specified {@link StringBuilder}.
    *
    * @param fullName provides the full name.
-   * @param errorMessageBuilder provides the {@link StringBuilder} to append 
+   * @param errorMessageBuilder provides the {@link StringBuilder} to append
    *                            error messages to.
    */
   public Name(String fullName, StringBuilder errorMessageBuilder)
@@ -60,12 +60,12 @@ public Name(String fullName, StringBuilder errorMessageBuilder)
   }
 
   /**
-   * Construct a new {@link Name} with the specified name and namespace, 
+   * Construct a new {@link Name} with the specified name and namespace,
    * and append errors in the specified {@link StringBuilder}.
    *
    * @param name provides the name.
-   * @param namespace provides the namespace.            
-   * @param errorMessageBuilder provides the {@link StringBuilder} to append 
+   * @param namespace provides the namespace.
+   * @param errorMessageBuilder provides the {@link StringBuilder} to append
    *                            error messages to.
    */
   public Name(String name, String namespace, StringBuilder errorMessageBuilder)
@@ -78,7 +78,7 @@ public Name(String name, String namespace, StringBuilder errorMessageBuilder)
    * append errors in the specified {@link StringBuilder}.
    *
    * @param fullName provides the full name.
-   * @param errorMessageBuilder provides the {@link StringBuilder} to append 
+   * @param errorMessageBuilder provides the {@link StringBuilder} to append
    *                            error messages to.
    */
   public boolean setName(String fullName, StringBuilder errorMessageBuilder)
@@ -111,12 +111,12 @@ public boolean setName(String fullName, StringBuilder errorMessageBuilder)
   }
 
   /**
-   * Sets this {@link Name} with the specified name and namespace, 
+   * Sets this {@link Name} with the specified name and namespace,
    * and append errors in the specified {@link StringBuilder}.
    *
    * @param name provides the name.
-   * @param namespace provides the namespace.            
-   * @param errorMessageBuilder provides the {@link StringBuilder} to append 
+   * @param namespace provides the namespace.
+   * @param errorMessageBuilder provides the {@link StringBuilder} to append
    *                            error messages to.
    */
   public boolean setName(String name, String namespace, StringBuilder errorMessageBuilder)
@@ -211,6 +211,11 @@ public static boolean isValidUnqualifiedName(String name)
     return UNQUALIFIED_NAME_PATTERN.matcher(name).matches();
   }
 
+  @Override
+  public int compareTo(Name o) {
+    return this.getFullName().compareTo(o.getFullName());
+  }
+
   private boolean _isEmpty = true;
   private boolean _hasError = false;
   private String _name = "";