Generate template for top-level unnamed data schema.

RB=585427
BUG=SI-2161
G=si-dev
R=gogenc,erli,kvidhani,sha
A=kvidhani

CHANGELOG

@@ -1,5 +1,11 @@
+2.11.3
+------
+
+
 2.11.2
 ------
+(RB=585427)
+Generate template for top-level unnamed data schema.
 
 2.11.1
 ------
@@ -20,7 +26,7 @@ Disallow @ActionParam on non-action methods.
 2.10.19
 -------
 (RB=587506)
-Fix a bug in ZKPersistentConnection where a ZKConnection shouldn't be reused 
+Fix a bug in ZKPersistentConnection where a ZKConnection shouldn't be reused
 after session expiration.
 
 2.10.18

generator/src/main/java/com/linkedin/pegasus/generator/DataSchemaParser.java

@@ -79,10 +79,9 @@ public DataSchemaResolver getSchemaResolver()
    * @return {@link ParseResult} for what were read.
    * @throws IOException if there are problems opening or deleting files.
    */
-  public ParseResult parseSources(String sources[])
-      throws IOException
+  public ParseResult parseSources(String sources[]) throws IOException
   {
-    final StringBuilder messageBuilder = new StringBuilder();
+    final ParseResult result = new ParseResult();
 
     try
     {
@@ -97,25 +96,31 @@ public ParseResult parseSources(String sources[])
             final List<File> sourceFilesInDirectory = FileUtil.listFiles(sourceFile, filter);
             for (File f : sourceFilesInDirectory)
             {
-              parseFile(f, messageBuilder);
+              parseFile(f, result);
+              result._sourceFiles.add(f);
             }
           }
-          else if (sourceFile.getName().endsWith(".jar"))
+          else
           {
-            final JarFile jarFile = new JarFile(sourceFile);
-            final Enumeration<JarEntry> entries = jarFile.entries();
-            while (entries.hasMoreElements())
+            if (sourceFile.getName().endsWith(".jar"))
             {
-              final JarEntry entry = entries.nextElement();
-              if (!entry.isDirectory() && entry.getName().endsWith(FileDataSchemaResolver.DEFAULT_EXTENSION))
+              final JarFile jarFile = new JarFile(sourceFile);
+              final Enumeration<JarEntry> entries = jarFile.entries();
+              while (entries.hasMoreElements())
               {
-                parseJarEntry(jarFile, entry, messageBuilder);
+                final JarEntry entry = entries.nextElement();
+                if (!entry.isDirectory() && entry.getName().endsWith(FileDataSchemaResolver.DEFAULT_EXTENSION))
+                {
+                  parseJarEntry(jarFile, entry, result);
+                }
               }
             }
-          }
-          else
-          {
-            parseFile(sourceFile, messageBuilder);
+            else
+            {
+              parseFile(sourceFile, result);
+            }
+
+            result._sourceFiles.add(sourceFile);
           }
         }
         else
@@ -124,29 +129,34 @@ else if (sourceFile.getName().endsWith(".jar"))
           final DataSchema schema = _schemaResolver.findDataSchema(source, errorMessage);
           if (schema == null)
           {
-            messageBuilder.append("File cannot be opened or schema name cannot be resolved: ").append(source).append("\n");
+            result._messageBuilder.append("File cannot be opened or schema name cannot be resolved: ").append(source).append("\n");
           }
           if (errorMessage.length() > 0)
           {
-            messageBuilder.append(errorMessage.toString());
+            result._messageBuilder.append(errorMessage.toString());
           }
         }
       }
 
-      if (messageBuilder.length() > 0)
+      if (result._messageBuilder.length() > 0)
       {
-        throw new IOException(messageBuilder.toString());
+        throw new IOException(result.getMessage());
       }
 
-      return new ParseResult(messageBuilder.toString());
+      for (Map.Entry<String, DataSchemaLocation> entry : _schemaResolver.nameToDataSchemaLocations().entrySet()) {
+        final DataSchema schema = _schemaResolver.bindings().get(entry.getKey());
+        result._schemaAndLocations.put(schema, entry.getValue());
+      }
+
+      return result;
     }
     catch (RuntimeException e)
     {
-      if (messageBuilder.length() > 0)
+      if (result._messageBuilder.length() > 0)
       {
         e = new RuntimeException("Unexpected " + e.getClass().getSimpleName() + " encountered.\n" +
                                      "This may be caused by the following parsing or processing errors:\n" +
-                                     messageBuilder, e);
+                                     result.getMessage(), e);
       }
       throw e;
     }
@@ -159,26 +169,31 @@ else if (sourceFile.getName().endsWith(".jar"))
    * @param messageBuilder {@link StringBuilder} to update message.
    * @throws IOException if there is a file access error.
    */
-  private void parseFile(File schemaSourceFile, StringBuilder messageBuilder)
+  private void parseFile(File schemaSourceFile, ParseResult result)
       throws IOException
   {
     final DataSchemaLocation location = getSchemaLocation(schemaSourceFile);
-    // Whether a source file has already been resolved to data schemas
+    // if a the data schema has been resolved before, must skip parsing again, because one name can't be bound to two data schemas
     if (_schemaResolver.locationResolved(location))
     {
       return;
     }
 
     final InputStream inputStream = new SchemaFileInputStream(schemaSourceFile);
-    final List<DataSchema> schemas = parseSchemaStream(inputStream, location, messageBuilder);
+    final List<DataSchema> schemas = parseSchemaStream(inputStream, location, result);
 
     for (DataSchema schema : schemas)
     {
-      validateSchemaWithPath(schemaSourceFile.getAbsolutePath(), schema);
+      if (schema instanceof NamedDataSchema)
+      {
+        validateSchemaWithPath(schemaSourceFile.getAbsolutePath(), (NamedDataSchema) schema);
+      }
+
+      result._schemaAndLocations.put(schema, location);
     }
   }
 
-  private void parseJarEntry(JarFile schemaJarFile, JarEntry jarEntry, StringBuilder messageBuilder)
+  private void parseJarEntry(JarFile schemaJarFile, JarEntry jarEntry, ParseResult result)
       throws IOException
   {
     final DataSchemaLocation location = getSchemaLocation(schemaJarFile, jarEntry.getName());
@@ -188,11 +203,16 @@ private void parseJarEntry(JarFile schemaJarFile, JarEntry jarEntry, StringBuild
     }
 
     final InputStream jarStream = schemaJarFile.getInputStream(jarEntry);
-    final List<DataSchema> schemas = parseSchemaStream(jarStream, location, messageBuilder);
+    final List<DataSchema> schemas = parseSchemaStream(jarStream, location, result);
 
     for (DataSchema schema : schemas)
     {
-      validateSchemaWithPath(location.toString(), schema);
+      if (schema instanceof NamedDataSchema)
+      {
+        validateSchemaWithPath(location.toString(), (NamedDataSchema) schema);
+      }
+
+      result._schemaAndLocations.put(schema, location);
     }
   }
 
@@ -209,37 +229,35 @@ private DataSchemaLocation getSchemaLocation(JarFile jarFile, String pathInJar)
   /**
    * Checks that the schema name and namespace match the file name and path.  These must match for FileDataSchemaResolver to find a schema pdscs by fully qualified name.
    */
-  private void validateSchemaWithPath(String path, DataSchema schema)
+  private void validateSchemaWithPath(String path, NamedDataSchema namedDataSchema)
   {
-    if (schema instanceof NamedDataSchema)
-    {
-      final NamedDataSchema namedDataSchema = (NamedDataSchema) schema;
-      final String namespace = namedDataSchema.getNamespace();
+    final String namespace = namedDataSchema.getNamespace();
 
-      if (!FileUtil.removeFileExtension(path.substring(path.lastIndexOf(File.separator) + 1)).equalsIgnoreCase(namedDataSchema.getName()))
-      {
-        throw new IllegalArgumentException(namedDataSchema.getFullName() + " has name that does not match path '" +
-                                               path + "'");
-      }
+    if (!FileUtil.removeFileExtension(path.substring(path.lastIndexOf(File.separator) + 1)).equalsIgnoreCase(namedDataSchema.getName()))
+    {
+      throw new IllegalArgumentException(namedDataSchema.getFullName() + " has name that does not match path '" +
+                                             path + "'");
+    }
 
-      final String parent = path.substring(0, path.lastIndexOf(File.separator));
-      if (!parent.endsWith(namespace.replace('.', File.separatorChar)))
-      {
-        throw new IllegalArgumentException(namedDataSchema.getFullName() + " has namespace that does not match " +
-                                               "parent path '" + parent + "'");
-      }
+    final String parent = path.substring(0, path.lastIndexOf(File.separator));
+    if (!parent.endsWith(namespace.replace('.', File.separatorChar)))
+    {
+      throw new IllegalArgumentException(namedDataSchema.getFullName() + " has namespace that does not match " +
+                                             "parent path '" + parent + "'");
     }
   }
 
   /**
    * Parse a source file to obtain the data schemas contained within.
+   * This method will cause the {@link DataSchemaResolver} to resolve any referenced named and unnamed schemas,
+   * as well as registering named schemas in its bindings.
    *
    * @param schemaInputStream provides the source data.
    * @param messageBuilder {@link StringBuilder} to update message.
-   * @return the data schemas within the source file.
+   * @return the top-level data schemas within the source file.
    * @throws IOException if there is a file access error.
    */
-  private List<DataSchema> parseSchemaStream(InputStream schemaInputStream, DataSchemaLocation schemaLocation, StringBuilder messageBuilder)
+  private List<DataSchema> parseSchemaStream(InputStream schemaInputStream, DataSchemaLocation schemaLocation, ParseResult result)
       throws IOException
   {
     final SchemaParser parser = new SchemaParser(_schemaResolver);
@@ -258,34 +276,25 @@ private List<DataSchema> parseSchemaStream(InputStream schemaInputStream, DataSc
       schemaInputStream.close();
       if (parser.hasError())
       {
-        messageBuilder.append(schemaLocation.toString())
-            .append(",").append(parser.errorMessage());
+        result._messageBuilder.append(schemaLocation.toString()).append(",").append(parser.errorMessage());
       }
     }
   }
 
   /**
    * Represent the result of schema parsing. Consist of two parts: schema from file path and from schema name, based on user input.
    * The two parts are mutually exclusive, and the union of two consists of all schema resolved.
+   *
+   * The result contains all resolved data schemas, both directly defined by the source files, or transitively referenced by the former.
+   * Both top-level and embedded named schemas are included. Only top-level unnamed schemas are included.
    */
-  public class ParseResult
+  public static class ParseResult
   {
-    private final Map<NamedDataSchema, DataSchemaLocation> _schemaAndLocations = new HashMap<NamedDataSchema, DataSchemaLocation>();
+    private final Map<DataSchema, DataSchemaLocation> _schemaAndLocations = new HashMap<DataSchema, DataSchemaLocation>();
     private final Set<File> _sourceFiles = new HashSet<File>();
-    private final String _message;
-
-    public ParseResult(String message)
-    {
-      for (Map.Entry<String, DataSchemaLocation> entry : _schemaResolver.nameToDataSchemaLocations().entrySet()) {
-        final NamedDataSchema schema = _schemaResolver.bindings().get(entry.getKey());
-        _schemaAndLocations.put(schema, entry.getValue());
-        _sourceFiles.add(entry.getValue().getSourceFile());
-      }
-
-      _message = message;
-    }
+    private final StringBuilder _messageBuilder = new StringBuilder();
 
-    public Map<NamedDataSchema, DataSchemaLocation> getSchemaAndLocations()
+    public Map<DataSchema, DataSchemaLocation> getSchemaAndLocations()
     {
       return _schemaAndLocations;
     }
@@ -297,7 +306,7 @@ public Set<File> getSourceFiles()
 
     public String getMessage()
     {
-      return _message;
+      return _messageBuilder.toString();
     }
   }
 

generator/src/main/java/com/linkedin/pegasus/generator/PegasusDataTemplateGenerator.java

@@ -123,7 +123,7 @@ public static GeneratorResult run(String resolverPath, String defaultPackage, fi
 
     final DataSchemaParser.ParseResult parseResult = schemaParser.parseSources(sources);
 
-    for (Map.Entry<NamedDataSchema, DataSchemaLocation> entry : parseResult.getSchemaAndLocations().entrySet())
+    for (Map.Entry<DataSchema, DataSchemaLocation> entry : parseResult.getSchemaAndLocations().entrySet())
     {
       specGenerator.generate(entry.getKey(), entry.getValue());
     }

generator/src/main/java/com/linkedin/pegasus/generator/TemplateSpecGenerator.java

@@ -789,7 +789,6 @@ private ClassInfo classInfoForUnnamed(ClassTemplateSpec enclosingClass, String n
       if (enclosingClass != null && classInfo.namespace.equals(enclosingClass.getFullName()))
       {
         classTemplateSpec.setEnclosingClass(enclosingClass);
-        //classTemplateSpec.setNamespace(enclosingClass.getNamespace());
         classTemplateSpec.setClassName(classInfo.name);
         classTemplateSpec.setModifiers(ModifierSpec.PUBLIC, ModifierSpec.STATIC, ModifierSpec.FINAL);
       }

gradle.properties

@@ -1,4 +1,4 @@
-version=2.11.1
+version=2.11.2
 sonatypeUsername=please_set_in_home_dir_if_uploading_to_maven_central
 sonatypePassword=please_set_in_home_dir_if_uploading_to_maven_central
 

restli-int-test-api/src/main/idl/com.linkedin.restli.examples.greetings.client.actions.restspec.json

@@ -133,4 +133,4 @@
       "returns" : "int"
     } ]
   }
-}
+}

restli-int-test-api/src/main/idl/com.linkedin.restli.examples.greetings.client.greetings.restspec.json

@@ -39,6 +39,15 @@
       "method" : "get_all"
     } ],
     "finders" : [ {
+      "name" : "empty",
+      "parameters" : [ {
+        "name" : "array",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.restli.examples.greetings.api.Empty\" }"
+      }, {
+        "name" : "map",
+        "type" : "{ \"type\" : \"map\", \"values\" : \"com.linkedin.restli.examples.greetings.api.Empty\" }"
+      } ]
+    }, {
       "name" : "search",
       "parameters" : [ {
         "name" : "tone",

restli-int-test-api/src/main/idl/com.linkedin.restli.examples.greetings.client.greetingsCallback.restspec.json

@@ -39,6 +39,15 @@
       "method" : "get_all"
     } ],
     "finders" : [ {
+      "name" : "empty",
+      "parameters" : [ {
+        "name" : "array",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.restli.examples.greetings.api.Empty\" }"
+      }, {
+        "name" : "map",
+        "type" : "{ \"type\" : \"map\", \"values\" : \"com.linkedin.restli.examples.greetings.api.Empty\" }"
+      } ]
+    }, {
       "name" : "search",
       "parameters" : [ {
         "name" : "tone",

restli-int-test-api/src/main/idl/com.linkedin.restli.examples.greetings.client.greetingsPromise.restspec.json

@@ -39,6 +39,15 @@
       "method" : "get_all"
     } ],
     "finders" : [ {
+      "name" : "empty",
+      "parameters" : [ {
+        "name" : "array",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.restli.examples.greetings.api.Empty\" }"
+      }, {
+        "name" : "map",
+        "type" : "{ \"type\" : \"map\", \"values\" : \"com.linkedin.restli.examples.greetings.api.Empty\" }"
+      } ]
+    }, {
       "name" : "search",
       "parameters" : [ {
         "name" : "tone",

restli-int-test-api/src/main/idl/com.linkedin.restli.examples.greetings.client.greetingsPromiseCtx.restspec.json

@@ -39,6 +39,15 @@
       "method" : "get_all"
     } ],
     "finders" : [ {
+      "name" : "empty",
+      "parameters" : [ {
+        "name" : "array",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.restli.examples.greetings.api.Empty\" }"
+      }, {
+        "name" : "map",
+        "type" : "{ \"type\" : \"map\", \"values\" : \"com.linkedin.restli.examples.greetings.api.Empty\" }"
+      } ]
+    }, {
       "name" : "search",
       "parameters" : [ {
         "name" : "tone",

restli-int-test-api/src/main/idl/com.linkedin.restli.examples.greetings.client.greetingsTask.restspec.json

@@ -39,6 +39,15 @@
       "method" : "get_all"
     } ],
     "finders" : [ {
+      "name" : "empty",
+      "parameters" : [ {
+        "name" : "array",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"com.linkedin.restli.examples.greetings.api.Empty\" }"
+      }, {
+        "name" : "map",
+        "type" : "{ \"type\" : \"map\", \"values\" : \"com.linkedin.restli.examples.greetings.api.Empty\" }"
+      } ]
+    }, {
       "name" : "search",
       "parameters" : [ {
         "name" : "tone",

restli-int-test-api/src/main/pegasus/com/linkedin/restli/examples/greetings/api/ArrayOfEmptys.pdsc

@@ -0,0 +1,5 @@
+{
+  "type" : "array",
+  "items" : "com.linkedin.restli.examples.greetings.api.Empty",
+  "doc" : "The name of this file is intentionally chosen to not follow naming convention and test if generator will correctly process"
+}