DRILL-6272: Refactor dynamic UDFs and function initializer tests to g…

…enerate needed binary and source jars at runtime

exec/java-exec/pom.xml

@@ -593,6 +593,54 @@
       <artifactId>netty-tcnative</artifactId>
       <classifier>${netty.tcnative.classifier}</classifier>
     </dependency>
+    <dependency>
+      <groupId>org.apache.maven</groupId>
+      <artifactId>maven-embedder</artifactId>
+      <version>3.5.3</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.maven</groupId>
+      <artifactId>maven-compat</artifactId>
+      <version>3.5.3</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.aether</groupId>
+      <artifactId>aether-connector-basic</artifactId>
+      <version>1.1.0</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.aether</groupId>
+      <artifactId>aether-transport-wagon</artifactId>
+      <version>1.1.0</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.maven.wagon</groupId>
+      <artifactId>wagon-http</artifactId>
+      <version>3.0.0</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.maven.wagon</groupId>
+      <artifactId>wagon-provider-api</artifactId>
+      <version>3.0.0</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
 
   <profiles>

exec/java-exec/src/test/java/org/apache/drill/exec/expr/fn/FunctionInitializerTest.java

@@ -18,21 +18,22 @@
 package org.apache.drill.exec.expr.fn;
 
 import com.google.common.collect.Lists;
+import org.apache.commons.io.FileUtils;
 import org.apache.drill.categories.SqlFunctionTest;
-import org.apache.drill.test.TestTools;
+import org.apache.drill.exec.udf.dynamic.JarBuilder;
+import org.apache.drill.test.BaseDirTestWatcher;
 import org.apache.drill.exec.util.JarUtil;
 import org.codehaus.janino.Java.CompilationUnit;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.runners.MockitoJUnitRunner;
-import org.mockito.stubbing.Answer;
 
+import java.io.File;
+import java.io.IOException;
 import java.net.URL;
 import java.net.URLClassLoader;
-import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -43,27 +44,32 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.apache.drill.test.TestTools.getResourceFile;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.spy;
 
-@RunWith(MockitoJUnitRunner.class)
 @Category(SqlFunctionTest.class)
 public class FunctionInitializerTest {
 
-  private static final String CLASS_NAME = "com.drill.udf.CustomLowerFunction";
+  @ClassRule
+  public static final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher();
+
+  private static final String CLASS_NAME = "org.apache.drill.udf.dynamic.CustomLowerFunction";
   private static URLClassLoader classLoader;
 
   @BeforeClass
   public static void init() throws Exception {
-    Path jars = TestTools.WORKING_PATH
-      .resolve(TestTools.TEST_RESOURCES)
-      .resolve("jars");
-    String binaryName = "DrillUDF-1.0.jar";
-    String sourceName = JarUtil.getSourceName(binaryName);
-    URL[] urls = {jars.resolve(binaryName).toUri().toURL(), jars.resolve(sourceName).toUri().toURL()};
+    File projectDir = dirTestWatcher.makeSubDir(Paths.get("drill-udf"));
+    FileUtils.copyDirectory(getResourceFile(Paths.get(projectDir.getName())), projectDir);
+    String binaryName = "drill-custom-lower";
+
+    JarBuilder jarBuilder = new JarBuilder();
+    int result = jarBuilder.build(binaryName, projectDir.getPath(), "**/CustomLowerFunction.java", null);
+    assertEquals("Build should be successful.", 0, result);
+
+    String binaryJar = binaryName + ".jar";
+    String jarsDir = Paths.get(projectDir.getPath(), "target").toString();
+    URL[] urls = {Paths.get(jarsDir, binaryJar).toUri().toURL(), Paths.get(jarsDir, JarUtil.getSourceName(binaryJar)).toUri().toURL()};
     classLoader = new URLClassLoader(urls);
   }
 
@@ -94,27 +100,21 @@ public void testGetMethod() {
 
   @Test
   public void testConcurrentFunctionBodyLoad() throws Exception {
-    final FunctionInitializer spyFunctionInitializer = spy(new FunctionInitializer(CLASS_NAME, classLoader));
     final AtomicInteger counter = new AtomicInteger();
-
-    doAnswer(new Answer<CompilationUnit>() {
+    final FunctionInitializer functionInitializer = new FunctionInitializer(CLASS_NAME, classLoader) {
       @Override
-      public CompilationUnit answer(InvocationOnMock invocation) throws Throwable {
+      CompilationUnit convertToCompilationUnit(Class<?> clazz) throws IOException {
         counter.incrementAndGet();
-        return (CompilationUnit) invocation.callRealMethod();
+        return super.convertToCompilationUnit(clazz);
       }
-    }).when(spyFunctionInitializer).convertToCompilationUnit(any(Class.class));
+    };
 
     int threadsNumber = 5;
     ExecutorService executor = Executors.newFixedThreadPool(threadsNumber);
 
     try {
-      List<Future<String>> results = executor.invokeAll(Collections.nCopies(threadsNumber, new Callable<String>() {
-        @Override
-        public String call() {
-          return spyFunctionInitializer.getMethod("eval");
-        }
-      }));
+      List<Future<String>> results = executor.invokeAll(Collections.nCopies(threadsNumber,
+        (Callable<String>) () -> functionInitializer.getMethod("eval")));
 
       final Set<String> uniqueResults = new HashSet<>();
       for (Future<String> result : results) {

exec/java-exec/src/test/java/org/apache/drill/exec/udf/dynamic/JarBuilder.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.udf.dynamic;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.ConsoleAppender;
+import org.apache.maven.cli.MavenCli;
+import org.apache.maven.cli.logging.Slf4jLogger;
+import org.codehaus.plexus.DefaultPlexusContainer;
+import org.codehaus.plexus.PlexusContainer;
+import org.codehaus.plexus.logging.BaseLoggerManager;
+import org.slf4j.LoggerFactory;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class JarBuilder {
+
+  private final MavenCli cli;
+
+  public JarBuilder() {
+    this.cli = new MavenCli() {
+      @Override
+      protected void customizeContainer(PlexusContainer container) {
+        ((DefaultPlexusContainer) container).setLoggerManager(new BaseLoggerManager() {
+          @Override
+          protected org.codehaus.plexus.logging.Logger createLogger(String s) {
+            return new Slf4jLogger(setupLogger(JarBuilder.class.getName(), Level.INFO));
+          }
+        });
+      }
+    };
+  }
+
+  /**
+   * Builds jars using embedded maven. Includes files / resources based given pattern,
+   * otherwise using defaults provided in pom.xml.
+   *
+   * @param jarName jar name
+   * @param projectDir project dir
+   * @param includeFiles pattern indicating which files should be included
+   * @param includeResources pattern indicating which resources should be included
+   *
+   * @return build exit code, 0 if build was successful
+   */
+  public int build(String jarName, String projectDir, String includeFiles, String includeResources) {
+    System.setProperty("maven.multiModuleProjectDirectory", projectDir);
+    List<String> params = new LinkedList<>();
+    params.add("clean");
+    params.add("package");
+    params.add("-DskipTests");
+    params.add("-Djar.finalName=" + jarName);
+    if (includeFiles != null) {
+      params.add("-Dinclude.files=" + includeFiles);
+    }
+    if (includeResources != null) {
+      params.add("-Dinclude.resources=" + includeResources);
+    }
+    return cli.doMain(params.toArray(new String[params.size()]), projectDir, System.out, System.err);
+  }
+
+  private static Logger setupLogger(String string, Level logLevel) {
+    LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
+    ConsoleAppender<ILoggingEvent> consoleAppender = new ConsoleAppender<>();
+    consoleAppender.setContext(loggerContext);
+    Logger logger = (Logger) LoggerFactory.getLogger(string);
+    logger.addAppender(consoleAppender);
+    logger.setLevel(logLevel);
+    return logger;
+  }
+
+}