Add Transport UDF plugin to apply default configurations and create wrapper codegen and test tasks

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Wed Aug 08 12:17:32 PDT 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-all.zip

gradlew

@@ -28,7 +28,7 @@ APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+DEFAULT_JVM_OPTS='"-Xmx64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"

gradlew.bat

@@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
 @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
+set DEFAULT_JVM_OPTS="-Xmx64m"
 
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome

settings.gradle

@@ -11,6 +11,7 @@ def modules = [
     'transportable-udfs-codegen',
     'transportable-udfs-compile-utils',
     'transportable-udfs-hive',
+    'transportable-udfs-plugin',
     'transportable-udfs-presto',
     'transportable-udfs-spark',
     'transportable-udfs-test:transportable-udfs-test-api',

transportable-udfs-annotation-processor/src/main/java/com/linkedin/transport/processor/Constants.java

@@ -9,7 +9,7 @@
 
 
 public class Constants {
-  public static final String UDF_RESOURCE_FILE_PATH = "META-INF/transport-udfs/udf-properties.json";
+  public static final String UDF_RESOURCE_FILE_PATH = "META-INF/transport-udfs/metadata.json";
 
   public static final String INTERFACE_NOT_IMPLEMENTED_ERROR =
       String.format(

transportable-udfs-annotation-processor/src/main/java/com/linkedin/transport/processor/TransportProcessor.java

@@ -91,7 +91,7 @@ public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment
 
   private void processImpl(RoundEnvironment roundEnv) {
     if (roundEnv.processingOver()) {
-      generateUDFPropertiesFile();
+      generateUDFMetadataFile();
     } else {
       processElements(roundEnv.getRootElements());
     }
@@ -178,18 +178,18 @@ private boolean typeElementOverridesTopLevelStdUDFMethods(TypeElement typeElemen
   }
 
   /**
-   * Generates the UDF properties resource file in a pretty-printed JSON format
+   * Generates the UDF metadata resource file in a pretty-printed JSON format
    */
-  private void generateUDFPropertiesFile() {
+  private void generateUDFMetadataFile() {
     Filer filer = processingEnv.getFiler();
     try {
       FileObject fileObject = filer.createResource(StandardLocation.CLASS_OUTPUT, "", Constants.UDF_RESOURCE_FILE_PATH);
       try (Writer writer = fileObject.openWriter()) {
         _transportUdfMetadata.toJson(writer);
       }
-      debug("Wrote Transport UDF properties file to: " + fileObject.toUri());
+      debug("Wrote Transport UDF metadata file to: " + fileObject.toUri());
     } catch (IOException e) {
-      fatalError(String.format("Unable to create UDF properties resource file: %s", e));
+      fatalError(String.format("Unable to create UDF metadata resource file: %s", e));
     }
   }
 

transportable-udfs-codegen/src/test/java/com/linkedin/transport/codegen/AbstractTestWrapperGenerator.java

@@ -44,38 +44,30 @@ void teardown() {
     _resourcesOutputDir.delete();
   }
 
-  void testWrapperGenerator(String udfPropertiesFileResource, String expectedSourcesOutputFolderResource) {
-    testWrapperGenerator(udfPropertiesFileResource, expectedSourcesOutputFolderResource, null);
+  void testWrapperGenerator(String udfMetadataFileResource, String expectedSourcesOutputFolderResource) {
+    testWrapperGenerator(udfMetadataFileResource, expectedSourcesOutputFolderResource, null);
   }
 
-  void testWrapperGenerator(String udfPropertiesFileResource, String expectedSourcesOutputFolderResource,
+  void testWrapperGenerator(String udfMetadataFileResource, String expectedSourcesOutputFolderResource,
       String expectedResourcesOutputFolderResource) {
     WrapperGeneratorContext context =
-        new WrapperGeneratorContext(TestUtils.getUDFPropertiesFromResource(udfPropertiesFileResource),
+        new WrapperGeneratorContext(TestUtils.getUDFMetadataFromResource(udfMetadataFileResource),
             _sourcesOutputDir, _resourcesOutputDir);
 
     getWrapperGenerator().generateWrappers(context);
 
-    final Path expectedSourcesOutputPath;
-    final Path expectedResourcesOutputPath;
     try {
-      expectedSourcesOutputPath = Paths.get(
+      Path expectedSourcesOutputPath = Paths.get(
           Thread.currentThread().getContextClassLoader().getResource(expectedSourcesOutputFolderResource).toURI());
+      TestUtils.assertDirectoriesAreEqual(_sourcesOutputDir.toPath(), expectedSourcesOutputPath);
       if (expectedResourcesOutputFolderResource != null) {
-        expectedResourcesOutputPath = Paths.get(
+        Path expectedResourcesOutputPath = Paths.get(
             Thread.currentThread().getContextClassLoader().getResource(expectedResourcesOutputFolderResource).toURI());
-      } else {
-        // No resources directory specified, so compare to an empty dir
-        File tmpEmptyResourcesDir = Files.createTempDir();
-        tmpEmptyResourcesDir.deleteOnExit();
-        expectedResourcesOutputPath = tmpEmptyResourcesDir.toPath();
+        TestUtils.assertDirectoriesAreEqual(_resourcesOutputDir.toPath(), expectedResourcesOutputPath);
       }
     } catch (URISyntaxException e) {
       throw new RuntimeException("Error constructing URI", e);
     }
-
-    TestUtils.assertDirectoriesAreEqual(_sourcesOutputDir.toPath(), expectedSourcesOutputPath);
-    TestUtils.assertDirectoriesAreEqual(_resourcesOutputDir.toPath(), expectedResourcesOutputPath);
   }
 
   abstract WrapperGenerator getWrapperGenerator();

transportable-udfs-codegen/src/test/java/com/linkedin/transport/codegen/TestHiveWrapperGenerator.java

@@ -17,6 +17,6 @@ WrapperGenerator getWrapperGenerator() {
 
   @Test
   public void testHiveWrapperGenerator() {
-    testWrapperGenerator("inputs/sample-udf-properties.json", "outputs/sample-udf-properties/hive/sources");
+    testWrapperGenerator("inputs/sample-udf-metadata.json", "outputs/sample-udf-metadata/hive/sources");
   }
 }

transportable-udfs-codegen/src/test/java/com/linkedin/transport/codegen/TestPrestoWrapperGenerator.java

@@ -17,7 +17,7 @@ WrapperGenerator getWrapperGenerator() {
 
   @Test
   public void testPrestoWrapperGenerator() {
-    testWrapperGenerator("inputs/sample-udf-properties.json", "outputs/sample-udf-properties/presto/sources",
-        "outputs/sample-udf-properties/presto/resources");
+    testWrapperGenerator("inputs/sample-udf-metadata.json", "outputs/sample-udf-metadata/presto/sources",
+        "outputs/sample-udf-metadata/presto/resources");
   }
 }

transportable-udfs-codegen/src/test/java/com/linkedin/transport/codegen/TestSparkWrapperGenerator.java

@@ -17,6 +17,6 @@ WrapperGenerator getWrapperGenerator() {
 
   @Test
   public void testSparkWrapperGenerator() {
-    testWrapperGenerator("inputs/sample-udf-properties.json", "outputs/sample-udf-properties/spark/sources");
+    testWrapperGenerator("inputs/sample-udf-metadata.json", "outputs/sample-udf-metadata/spark/sources");
   }
 }

transportable-udfs-codegen/src/test/java/com/linkedin/transport/codegen/TestUtils.java

@@ -21,12 +21,12 @@ class TestUtils {
   private TestUtils() {
   }
 
-  static TransportUDFMetadata getUDFPropertiesFromResource(String resource) {
+  static TransportUDFMetadata getUDFMetadataFromResource(String resource) {
     try (InputStreamReader reader = new InputStreamReader(
         Thread.currentThread().getContextClassLoader().getResourceAsStream(resource))) {
       return TransportUDFMetadata.fromJson(reader);
     } catch (IOException e) {
-      throw new RuntimeException("Could not read UDF properties from resource: " + resource, e);
+      throw new RuntimeException("Could not read UDF metadata from resource: " + resource, e);
     }
   }
 

transportable-udfs-examples/gradlew

@@ -28,7 +28,7 @@ APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+DEFAULT_JVM_OPTS='"-Xmx64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"

transportable-udfs-examples/gradlew.bat

@@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
 @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
+set DEFAULT_JVM_OPTS="-Xmx64m"
 
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome

transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle

@@ -1,67 +1,31 @@
+buildscript {
+  repositories {
+    mavenCentral()
+  }
+  dependencies {
+    classpath('com.linkedin.transport:transportable-udfs-plugin')
+  }
+}
+
 apply plugin: 'java'
+apply plugin: 'com.linkedin.transport.plugin'
 
 dependencies {
-  compile('com.linkedin.transport:transportable-udfs-api')
   // TODO: Reference all external dependencies from a single gradle file
   compile('com.google.guava:guava:24.1-jre')
   compile('org.apache.commons:commons-io:1.3.2')
-  testCompile('com.linkedin.transport:transportable-udfs-test-api')
 }
 
-// ============================================================================
-// transportable-udfs-plugin cannot be applied to this module as the plugin is also
-// built in this project. In projects using the Transportable UDF framework, the
-// following code will be applied by the plugin
-// ============================================================================
-
-dependencies {
-  annotationProcessor('com.linkedin.transport:transportable-udfs-annotation-processor')
+// If the license plugin is applied, disable license checks for the autogenerated source sets
+plugins.withId('com.github.hierynomus.license') {
+  licenseHive.enabled = false
+  licensePresto.enabled = false
+  licenseSpark.enabled = false
 }
 
 // TODO: Add a debugPlatform flag to allow debugging specific test methods in IntelliJ
 // for a particular platform other than default
 
-configurations {
-  platformTestRuntime { extendsFrom testRuntime }
-  hiveTestRuntime { extendsFrom platformTestRuntime }
-  prestoTestRuntime { extendsFrom platformTestRuntime }
-  sparkTestRuntime { extendsFrom platformTestRuntime }
-}
-
-dependencies {
-  testRuntimeOnly('com.linkedin.transport:transportable-udfs-test-generic')
-  platformTestRuntime sourceSets.main.output, sourceSets.test.output
-  hiveTestRuntime('com.linkedin.transport:transportable-udfs-test-hive')
-  prestoTestRuntime('com.linkedin.transport:transportable-udfs-test-presto')
-  sparkTestRuntime('com.linkedin.transport:transportable-udfs-test-spark')
-}
-
-task hiveTest(type: Test, dependsOn: test) {
-  group 'Verification'
-  description 'Runs the Hive tests.'
-  testClassesDirs = sourceSets.test.output.classesDirs
-  classpath = configurations.hiveTestRuntime
-  useTestNG()
-}
-
-task prestoTest(type: Test, dependsOn: test) {
-  group 'Verification'
-  description 'Runs the Presto tests.'
-  testClassesDirs = sourceSets.test.output.classesDirs
-  classpath = configurations.prestoTestRuntime
-  useTestNG()
-}
-
-task sparkTest(type: Test, dependsOn: test) {
-  group 'Verification'
-  description 'Runs the Spark tests.'
-  testClassesDirs = sourceSets.test.output.classesDirs
-  classpath = configurations.sparkTestRuntime
-  useTestNG()
-}
-
-check.dependsOn(hiveTest, prestoTest, sparkTest)
-
 prestoTest {
   exclude '**/TestArrayFillFunctionFailsOnPresto.class'
   exclude '**/TestStructCreateByIndexFunctionFailsOnPresto.class'

transportable-udfs-plugin/build.gradle

@@ -0,0 +1,36 @@
+plugins {
+  id 'java'
+  id 'java-gradle-plugin'
+}
+
+gradlePlugin {
+  plugins {
+    simplePlugin {
+      id = 'com.linkedin.transport.plugin'
+      implementationClass = 'com.linkedin.transport.plugin.TransportPlugin'
+    }
+  }
+}
+
+dependencies {
+  compile project(':transportable-udfs-api')
+  compile project(':transportable-udfs-codegen')
+  compile ('com.google.guava:guava:24.1-jre')
+  compile ('com.google.code.gson:gson:2.8.5')
+  testCompile('org.spockframework:spock-core:1.1-groovy-2.4') {
+    exclude group: 'org.codehaus.groovy'
+  }
+}
+
+def writeVersionInfo = { file ->
+  ant.propertyfile(file: file) {
+    entry(key: "transport-version", value: version)
+    entry(key: "hive-version", value: '1.2.2')
+    entry(key: "presto-version", value: '0.203')
+    entry(key: "spark-version", value: '2.3.0')
+  }
+}
+
+processResources.doLast {
+  writeVersionInfo(new File(sourceSets.main.output.resourcesDir, "version-info.properties"))
+}

transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/ConfigurationType.java

@@ -0,0 +1,13 @@
+/**
+ * Copyright 2019 LinkedIn Corporation. All rights reserved.
+ * Licensed under the BSD-2 Clause license.
+ * See LICENSE in the project root for license information.
+ */
+package com.linkedin.transport.plugin;
+
+public enum ConfigurationType {
+  ANNOTATION_PROCESSOR,
+  COMPILE_ONLY,
+  IMPLEMENTATION,
+  RUNTIME_ONLY
+}