From 8e2cafb21f6d149dc3bec7a3c7a845d54e678672 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Fri, 23 Oct 2015 17:21:31 +0900 Subject: [PATCH 01/72] base --- .../function/TestHiveFunctionLoader.java | 41 ++++++++ tajo-core/pom.xml | 5 + .../engine/function/HiveFunctionLoader.java | 94 +++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java new file mode 100644 index 0000000000..5a06d812b6 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java @@ -0,0 +1,41 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function; + +import com.google.common.collect.Lists; +import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.util.StringUtils; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.apache.tajo.LocalTajoTestingUtility.getResultText; +import static org.junit.Assert.assertEquals; + + +public class TestHiveFunctionLoader { + + @Test + public void testLoadHiveFunction() throws IOException { + HiveFunctionLoader.loadHiveUDFs(new TajoConf()); + } +} \ No newline at end of file diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index ed006a7191..10e446ac98 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -305,6 +305,11 @@ com.jayway.jsonpath json-path + + org.reflections + reflections + 0.9.10 + diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java new file mode 100644 index 0000000000..cc2a1350c4 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java @@ -0,0 +1,94 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.tajo.conf.TajoConf; +import org.reflections.Reflections; +import org.reflections.util.ConfigurationBuilder; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Enumeration; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +public class HiveFunctionLoader { + public static void loadHiveUDFs(TajoConf conf) { + String udfdir = conf.get("hive.udf.dir", "lib/hiveudf"); + //Reflections refl = new Reflections(new ConfigurationBuilder().setUrls()); + + try { + FileSystem localFS = FileSystem.getLocal(conf); + Path udfPath = new Path(udfdir); + + if (!localFS.isDirectory(udfPath)) { + return; + } + + for (FileStatus fstatus: localFS.listStatus(udfPath, (Path path)->path.getName().endsWith(".jar"))) { + + String absPath = fstatus.getPath().toString(); + + // Eliminate "file:" + if (absPath.startsWith("file:")) { + absPath = absPath.substring(5); + } + + JarFile jar = new JarFile(absPath); + Enumeration e = jar.entries(); + URL[] urls = getURLs(fstatus); + + while (e.hasMoreElements()) { + Set> UDFclasses = getClassesFromJarEntry(e.nextElement(), urls, UDF.class); + Set> GenericUDFclasses = getClassesFromJarEntry(e.nextElement(), urls, GenericUDF.class); + } + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + private static URL [] getURLs(FileStatus fstatus) throws MalformedURLException { + URL fileURL = fstatus.getPath().toUri().toURL(); + String jarURL = "jar:" + fileURL + "!/"; + + return new URL [] { new URL(jarURL) }; + } + + private static Set> getClassesFromJarEntry(JarEntry entry, URL [] urls, Class targetCls) { + String name = entry.getName(); + if (entry.isDirectory() || !name.endsWith(".class")) { + return null; + } + + Reflections refl = new Reflections(new ConfigurationBuilder(). + setUrls(urls). + addClassLoader(new URLClassLoader(urls))); + + return refl.getSubTypesOf(targetCls); + } +} From 070f3610442199edaa55e7f76c5343c549a66a01 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 27 Oct 2015 16:41:41 +0900 Subject: [PATCH 02/72] Method name refactoring --- .../apache/tajo/engine/function/HiveFunctionLoader.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java index cc2a1350c4..2a871c5d4f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java @@ -39,7 +39,6 @@ public class HiveFunctionLoader { public static void loadHiveUDFs(TajoConf conf) { String udfdir = conf.get("hive.udf.dir", "lib/hiveudf"); - //Reflections refl = new Reflections(new ConfigurationBuilder().setUrls()); try { FileSystem localFS = FileSystem.getLocal(conf); @@ -63,8 +62,8 @@ public static void loadHiveUDFs(TajoConf conf) { URL[] urls = getURLs(fstatus); while (e.hasMoreElements()) { - Set> UDFclasses = getClassesFromJarEntry(e.nextElement(), urls, UDF.class); - Set> GenericUDFclasses = getClassesFromJarEntry(e.nextElement(), urls, GenericUDF.class); + Set> UDFclasses = getSubclassesFromJarEntry(e.nextElement(), urls, UDF.class); + Set> GenericUDFclasses = getSubclassesFromJarEntry(e.nextElement(), urls, GenericUDF.class); } } } catch (IOException e) { @@ -79,7 +78,7 @@ public static void loadHiveUDFs(TajoConf conf) { return new URL [] { new URL(jarURL) }; } - private static Set> getClassesFromJarEntry(JarEntry entry, URL [] urls, Class targetCls) { + private static Set> getSubclassesFromJarEntry(JarEntry entry, URL[] urls, Class targetCls) { String name = entry.getName(); if (entry.isDirectory() || !name.endsWith(".class")) { return null; From 7efc29eefe6d78837cc6a817d1f1c68d0f550fce Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 27 Oct 2015 16:55:08 +0900 Subject: [PATCH 03/72] Remove a simple method --- .../apache/tajo/engine/function/HiveFunctionLoader.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java index 2a871c5d4f..580cc74cfc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java @@ -59,7 +59,7 @@ public static void loadHiveUDFs(TajoConf conf) { JarFile jar = new JarFile(absPath); Enumeration e = jar.entries(); - URL[] urls = getURLs(fstatus); + URL[] urls = new URL [] { new URL("jar:" + fstatus.getPath().toUri().toURL() + "!/") }; while (e.hasMoreElements()) { Set> UDFclasses = getSubclassesFromJarEntry(e.nextElement(), urls, UDF.class); @@ -71,13 +71,6 @@ public static void loadHiveUDFs(TajoConf conf) { } } - private static URL [] getURLs(FileStatus fstatus) throws MalformedURLException { - URL fileURL = fstatus.getPath().toUri().toURL(); - String jarURL = "jar:" + fileURL + "!/"; - - return new URL [] { new URL(jarURL) }; - } - private static Set> getSubclassesFromJarEntry(JarEntry entry, URL[] urls, Class targetCls) { String name = entry.getName(); if (entry.isDirectory() || !name.endsWith(".class")) { From 3a803f19a7758b18d5a592a4f44d95c071cf6f14 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 29 Oct 2015 12:21:41 +0900 Subject: [PATCH 04/72] get UDF classes --- .../engine/function/HiveFunctionLoader.java | 56 +++++++++++-------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java index 580cc74cfc..cb973459ae 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java @@ -21,20 +21,23 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.function.FunctionInvocation; +import org.apache.tajo.function.FunctionSignature; +import org.apache.tajo.function.FunctionSupplement; import org.reflections.Reflections; import org.reflections.util.ConfigurationBuilder; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; -import java.util.Enumeration; +import java.util.LinkedList; +import java.util.List; import java.util.Set; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; public class HiveFunctionLoader { public static void loadHiveUDFs(TajoConf conf) { @@ -48,39 +51,46 @@ public static void loadHiveUDFs(TajoConf conf) { return; } - for (FileStatus fstatus: localFS.listStatus(udfPath, (Path path)->path.getName().endsWith(".jar"))) { + for (FileStatus fstatus : localFS.listStatus(udfPath, (Path path) -> path.getName().endsWith(".jar"))) { - String absPath = fstatus.getPath().toString(); + URL[] urls = new URL[]{new URL("jar:" + fstatus.getPath().toUri().toURL() + "!/")}; - // Eliminate "file:" - if (absPath.startsWith("file:")) { - absPath = absPath.substring(5); - } - - JarFile jar = new JarFile(absPath); - Enumeration e = jar.entries(); - URL[] urls = new URL [] { new URL("jar:" + fstatus.getPath().toUri().toURL() + "!/") }; + List udfList = new LinkedList<>(); - while (e.hasMoreElements()) { - Set> UDFclasses = getSubclassesFromJarEntry(e.nextElement(), urls, UDF.class); - Set> GenericUDFclasses = getSubclassesFromJarEntry(e.nextElement(), urls, GenericUDF.class); + // For UDF's decendants (legacy) + Set> udfClasses = getSubclassesFromJarEntry(urls, UDF.class); + if (udfClasses != null) { + analyzeUDFclasses(udfClasses, udfList); } + + // For GenericUDF's decendants (newer interface) + Set> genericUDFclasses = getSubclassesFromJarEntry(urls, GenericUDF.class); } } catch (IOException e) { e.printStackTrace(); } } - private static Set> getSubclassesFromJarEntry(JarEntry entry, URL[] urls, Class targetCls) { - String name = entry.getName(); - if (entry.isDirectory() || !name.endsWith(".class")) { - return null; - } - + private static Set> getSubclassesFromJarEntry(URL[] urls, Class targetCls) { Reflections refl = new Reflections(new ConfigurationBuilder(). setUrls(urls). addClassLoader(new URLClassLoader(urls))); return refl.getSubTypesOf(targetCls); } + + private static void analyzeUDFclasses(Set> classes, List list) { + for (Class clazz: classes) { + String name; + FunctionSignature signature; + FunctionInvocation invocation = new FunctionInvocation(); + FunctionSupplement supplement = new FunctionSupplement(); + + Description desc = clazz.getDeclaredAnnotation(Description.class); + + name = desc == null ? clazz.getSimpleName() : desc.name(); + + System.out.println(name); + } + } } From 0fdfa0903f5d4922bf5d1c484013ec079770dede Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 29 Oct 2015 12:22:07 +0900 Subject: [PATCH 05/72] refactor for Java 8 --- .../tajo/engine/function/FunctionLoader.java | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java index 18baf0f89c..2284d52e5d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathFilter; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.FunctionDesc; @@ -110,12 +109,8 @@ public static Map loadUserDefinedFunctions(Tajo List filePaths = TUtil.newList(); if (localFS.isDirectory(codePath)) { - for (FileStatus file : localFS.listStatus(codePath, new PathFilter() { - @Override - public boolean accept(Path path) { - return path.getName().endsWith(PythonScriptEngine.FILE_EXTENSION); - } - })) { + for (FileStatus file : localFS.listStatus(codePath, + (Path path) -> path.getName().endsWith(PythonScriptEngine.FILE_EXTENSION))) { filePaths.add(file.getPath()); } } else { @@ -135,12 +130,8 @@ public boolean accept(Path path) { public static Set findScalarFunctions() { Set functions = Sets.newHashSet(); - Set scalarFunctions = findPublicStaticMethods("org.apache.tajo.engine.function", new Predicate() { - @Override - public boolean evaluate(Object object) { - return ((Method) object).getAnnotation(ScalarFunction.class) != null; - } - }); + Set scalarFunctions = findPublicStaticMethods("org.apache.tajo.engine.function", + (Object object) -> ((Method)object).getAnnotation(ScalarFunction.class) != null); for (Method method : scalarFunctions) { ScalarFunction annotation = method.getAnnotation(ScalarFunction.class); @@ -170,12 +161,8 @@ private static boolean isPublicStaticMethod(Method method) { } private static Set findFunctionCollections(String packageName) { - return ClassUtil.findClasses(null, packageName, new Predicate() { - @Override - public boolean evaluate(Object object) { - return ((Class)object).getAnnotation(FunctionCollection.class) != null; - } - }); + return ClassUtil.findClasses(null, packageName, + (Object object)->((Class)object).getAnnotation(FunctionCollection.class) != null); } private static Collection buildFunctionDescs(ScalarFunction annotation, Method method) { From 1cd0d5a04b88d6fc2b7bf11f7c85089a7eef375b Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 29 Oct 2015 12:23:03 +0900 Subject: [PATCH 06/72] test for local --- .../apache/tajo/engine/function/TestHiveFunctionLoader.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java index 5a06d812b6..73f107314e 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java @@ -36,6 +36,8 @@ public class TestHiveFunctionLoader { @Test public void testLoadHiveFunction() throws IOException { - HiveFunctionLoader.loadHiveUDFs(new TajoConf()); + TajoConf conf = new TajoConf(); + conf.set("hive.udf.dir", "/Users/eminency/test/hiveudf"); + HiveFunctionLoader.loadHiveUDFs(conf); } } \ No newline at end of file From 98f65a6001b4619e12ee34bc95a13c367b5ad1a9 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 29 Oct 2015 16:11:12 +0900 Subject: [PATCH 07/72] name and deterministic --- .../engine/function/HiveFunctionLoader.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java index cb973459ae..709cc6c960 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.conf.TajoConf; @@ -81,16 +82,27 @@ private static Set> getSubclassesFromJarEntry(URL[] urls, private static void analyzeUDFclasses(Set> classes, List list) { for (Class clazz: classes) { - String name; + String [] names; + boolean deterministic = true; FunctionSignature signature; FunctionInvocation invocation = new FunctionInvocation(); FunctionSupplement supplement = new FunctionSupplement(); Description desc = clazz.getDeclaredAnnotation(Description.class); + if (desc != null) { + names = desc.name().split(","); + for (int i=0; i Date: Thu, 29 Oct 2015 16:11:28 +0900 Subject: [PATCH 08/72] variable name refactoring --- .../src/main/java/org/apache/tajo/catalog/FunctionDesc.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java index 5fa0c15f7a..d1306f9717 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java @@ -46,9 +46,9 @@ public class FunctionDesc implements ProtoObject, Cloneable, public FunctionDesc() { } - public FunctionDesc(String signature, Class clazz, + public FunctionDesc(String funcName, Class clazz, FunctionType funcType, DataType retType, @NotNull DataType [] params) { - this.signature = new FunctionSignature(funcType, signature.toLowerCase(), retType, params); + this.signature = new FunctionSignature(funcType, funcName.toLowerCase(), retType, params); this.invocation = new FunctionInvocation(); this.invocation.setLegacy(new ClassBaseInvocationDesc<>(clazz)); this.supplement = new FunctionSupplement(); From a0fe3b21114ab67f710585485891ac787782e964 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 29 Oct 2015 16:29:55 +0900 Subject: [PATCH 09/72] remove useless enum --- .../engine/function/{ => hiveudf}/HiveFunctionLoader.java | 2 +- .../java/org/apache/tajo/plan/function/GeneralFunction.java | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) rename tajo-core/src/main/java/org/apache/tajo/engine/function/{ => hiveudf}/HiveFunctionLoader.java (98%) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java rename to tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index 709cc6c960..9baf6fc39a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.function; +package org.apache.tajo.engine.function.hiveudf; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java index 39db5c6a33..e68ee3166b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java @@ -42,11 +42,6 @@ public void init(OverridableConf queryContext, FunctionEval.ParamType [] paramTy public abstract Datum eval(Tuple params); - public enum Type { - AGG, - GENERAL - } - @Override public String toJson() { return CatalogGsonHelper.toJson(this, GeneralFunction.class); From 6dcd24b537a0b245a079ef8db1c411a0a7563b19 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 29 Oct 2015 16:31:32 +0900 Subject: [PATCH 10/72] Revert "remove useless enum" This reverts commit a8739670ffb688b3d2c2ea5b8e8422d21059cf92. --- .../engine/function/{hiveudf => }/HiveFunctionLoader.java | 2 +- .../java/org/apache/tajo/plan/function/GeneralFunction.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) rename tajo-core/src/main/java/org/apache/tajo/engine/function/{hiveudf => }/HiveFunctionLoader.java (98%) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java rename to tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java index 9baf6fc39a..709cc6c960 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/HiveFunctionLoader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.function.hiveudf; +package org.apache.tajo.engine.function; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java index e68ee3166b..39db5c6a33 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java @@ -42,6 +42,11 @@ public void init(OverridableConf queryContext, FunctionEval.ParamType [] paramTy public abstract Datum eval(Tuple params); + public enum Type { + AGG, + GENERAL + } + @Override public String toJson() { return CatalogGsonHelper.toJson(this, GeneralFunction.class); From 055a6db84feeb185c279b1b41e61190ede0ca155 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 29 Oct 2015 16:32:30 +0900 Subject: [PATCH 11/72] Remove useless enum --- .../java/org/apache/tajo/plan/function/GeneralFunction.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java index 39db5c6a33..e68ee3166b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java @@ -42,11 +42,6 @@ public void init(OverridableConf queryContext, FunctionEval.ParamType [] paramTy public abstract Datum eval(Tuple params); - public enum Type { - AGG, - GENERAL - } - @Override public String toJson() { return CatalogGsonHelper.toJson(this, GeneralFunction.class); From 64899ae7bdb6b2dad951c3d98d9282ecc83bd4cc Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 9 Nov 2015 11:58:20 +0900 Subject: [PATCH 12/72] Weird variable name changed --- .../org/apache/tajo/engine/eval/ExprTestBase.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 600a45f38d..c416b8e349 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -70,7 +70,7 @@ import static org.junit.Assert.*; public class ExprTestBase { - private static TajoTestingCluster util; + private static TajoTestingCluster cluster; private static TajoConf conf; private static CatalogService cat; private static SQLAnalyzer analyzer; @@ -88,10 +88,10 @@ public ExprTestBase() { @BeforeClass public static void setUp() throws Exception { - util = new TajoTestingCluster(); - conf = util.getConfiguration(); - util.startCatalogCluster(); - cat = util.getCatalogService(); + cluster = new TajoTestingCluster(); + conf = cluster.getConfiguration(); + cluster.startCatalogCluster(); + cat = cluster.getCatalogService(); cat.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234/warehouse"); cat.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); Map map = FunctionLoader.load(); @@ -103,13 +103,13 @@ public static void setUp() throws Exception { analyzer = new SQLAnalyzer(); preLogicalPlanVerifier = new PreLogicalPlanVerifier(cat); planner = new LogicalPlanner(cat, TablespaceManager.getInstance()); - optimizer = new LogicalOptimizer(util.getConfiguration(), cat, TablespaceManager.getInstance()); + optimizer = new LogicalOptimizer(cluster.getConfiguration(), cat, TablespaceManager.getInstance()); annotatedPlanVerifier = new LogicalPlanVerifier(); } @AfterClass public static void tearDown() throws Exception { - util.shutdownCatalogCluster(); + cluster.shutdownCatalogCluster(); } private static void assertJsonSerDer(EvalNode expr) { From 935cd2891d6810d466c12838cf0498f94eb47bb0 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 9 Nov 2015 14:32:48 +0900 Subject: [PATCH 13/72] analyzeUDF test --- .../function/TestHiveFunctionLoader.java | 43 -------- .../engine/function/hiveudf/HiveUDFtest.java | 19 ++++ .../hiveudf/TestHiveFunctionLoader.java | 68 ++++++++++++ .../src/test/resources/hiveudf/myudf.jar | Bin 0 -> 1616 bytes .../{ => hiveudf}/HiveFunctionLoader.java | 99 ++++++++++++++---- 5 files changed, 165 insertions(+), 64 deletions(-) delete mode 100644 tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java create mode 100644 tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java create mode 100644 tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java create mode 100644 tajo-core-tests/src/test/resources/hiveudf/myudf.jar rename tajo-core/src/main/java/org/apache/tajo/engine/function/{ => hiveudf}/HiveFunctionLoader.java (51%) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java deleted file mode 100644 index 73f107314e..0000000000 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestHiveFunctionLoader.java +++ /dev/null @@ -1,43 +0,0 @@ -/*** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.function; - -import com.google.common.collect.Lists; -import org.apache.tajo.catalog.FunctionDesc; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.util.StringUtils; -import org.junit.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; - -import static org.apache.tajo.LocalTajoTestingUtility.getResultText; -import static org.junit.Assert.assertEquals; - - -public class TestHiveFunctionLoader { - - @Test - public void testLoadHiveFunction() throws IOException { - TajoConf conf = new TajoConf(); - conf.set("hive.udf.dir", "/Users/eminency/test/hiveudf"); - HiveFunctionLoader.loadHiveUDFs(conf); - } -} \ No newline at end of file diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java new file mode 100644 index 0000000000..996db84f4e --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java @@ -0,0 +1,19 @@ +package org.apache.tajo.engine.function.hiveudf; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; + +@Description(name="multiplestr") +public class HiveUDFtest extends UDF { + Text evaluate(Text str, IntWritable num) { + String origin = str.toString(); + + for (int i=0; i_xcQ(l^q2DU+jfrEns!^KLtwD?x~%w=_ThR78lnBp7vvM6j<|Y>yCB$ zEH#fGvrMx1nAY^M>Eo8FKu?>#(;UfSY}Z-8uK%*AQ!UQ-k4Ex)C!35EzPq-jp;zzD zJiq>WIm7N7n-Ui$EX+`DWR>oY=vtM^tt}SDIkWce)rX9Vk?Tx2R_@BwI@Ggz_p2rM zcZM9Cx#x@I$(_e0*EzFU?dm@ea^u?emPxN?_62<3bouktjMt4qf1gdYs4F--mn*Af zjg-fJedlR!rp5|yX?f$h{%p)mlV$2#1k+#HtqDE0N3bJ$ZBOcr<;&zNUU|45wp{$r z`g)OqY>J^{#INwT3B0P@uldXZx^B#FxmuVlxb5Gpqf4acyPgt$-tNAqD#P!vLwR&d z&CW+|n>pgkqLVs4UD&+e@K8#|Be~Mb)7>Jc>Qkq(nN57SzSoaQWbN^b&t^aHnREHJ z{(|K{DkmkYu^AUwSUEE-GdMeQzw~D1kA{=ZuybAIU-I%JZ)f)_<4NZ|_A31Sk(_pX ziO%ht8xspPq$J(0$`pN+`Mgo>KW~6HJI9%un-i-T85kxoK~fdTF&SD=0E@}ipjdwq zLlN7%{xWPaf*~vu^$Xd0dh^vJgqaV{;IY|oTFq|S&0W#P*0SkVv%kha)LPwsvDa_94Vm%se(KOfI%b9i05nw-hTXA7I!<*sc^44>_rW4iI|g^Gni zsc}Xj%^OY!SYNO`SzsKr(nhoWxzYQM&n**LJ&r5A?25Vhmu1b&MlFeJiz4`U%5*1A z^RNq0K6(0{b={wjI;T_ioi|W+SJG@G*GnWR>j?!wovLG(1^3=6Cxfx$}H75mnWLM>++?s6LX35U}_T0@ftgWU|8%%_-*9N2H}kJn!ZkV9{<1kni(~HZW%~W3FJ042=Eddw z#mWhMpQpThxzJS8&TZ}PWK1==LG~qqPu_l6B zR!2uDoaD;eJO2 zAm loadHiveUDFs(TajoConf conf) { String udfdir = conf.get("hive.udf.dir", "lib/hiveudf"); + ArrayList funcList = new ArrayList<>(); try { FileSystem localFS = FileSystem.getLocal(conf); Path udfPath = new Path(udfdir); if (!localFS.isDirectory(udfPath)) { - return; + return null; } for (FileStatus fstatus : localFS.listStatus(udfPath, (Path path) -> path.getName().endsWith(".jar"))) { URL[] urls = new URL[]{new URL("jar:" + fstatus.getPath().toUri().toURL() + "!/")}; - List udfList = new LinkedList<>(); - // For UDF's decendants (legacy) Set> udfClasses = getSubclassesFromJarEntry(urls, UDF.class); if (udfClasses != null) { - analyzeUDFclasses(udfClasses, udfList); + analyzeUDFclasses(udfClasses, funcList); } // For GenericUDF's decendants (newer interface) Set> genericUDFclasses = getSubclassesFromJarEntry(urls, GenericUDF.class); + if (genericUDFclasses != null) { + analyzeGenericUDFclasses(genericUDFclasses, funcList); + } + } } catch (IOException e) { e.printStackTrace(); } + + return funcList; } private static Set> getSubclassesFromJarEntry(URL[] urls, Class targetCls) { @@ -80,15 +84,12 @@ private static Set> getSubclassesFromJarEntry(URL[] urls, return refl.getSubTypesOf(targetCls); } - private static void analyzeUDFclasses(Set> classes, List list) { + static void analyzeUDFclasses(Set> classes, List list) { for (Class clazz: classes) { String [] names; boolean deterministic = true; - FunctionSignature signature; - FunctionInvocation invocation = new FunctionInvocation(); - FunctionSupplement supplement = new FunctionSupplement(); - Description desc = clazz.getDeclaredAnnotation(Description.class); + Description desc = clazz.getAnnotation(Description.class); if (desc != null) { names = desc.name().split(","); for (int i=0; i> classes, List 0) { + params = new TajoDataTypes.DataType[hiveUDFparams.length]; + for (int i=0; i> classes, ArrayList list) { + } + + private static TajoDataTypes.DataType convertHiveTypeToTajoType(Class hiveType) { + if (hiveType == null) + return null; + + if (hiveType == IntWritable.class) { + return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4); + } + if (hiveType == LongWritable.class) { + return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT8); + } + if (hiveType == Text.class) { + return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT); + } + if (hiveType == FloatWritable.class) { + return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT4); + } + if (hiveType == DoubleWritable.class) { + return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT8); + } + + return null; + } } From 6bc4e1eba76f242ee26c0dff2143dff819accb1b Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 9 Nov 2015 14:33:08 +0900 Subject: [PATCH 14/72] test code missed --- .../hiveudf/TestHiveFunctionLoader.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 8c419db271..280196235f 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -19,21 +19,20 @@ package org.apache.tajo.engine.function.hiveudf; import com.google.common.base.Preconditions; +import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.catalog.CatalogClient; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.eval.ExprTestBase; -import org.apache.tajo.engine.function.hiveudf.HiveFunctionLoader; import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.io.IOException; +import static org.junit.Assert.assertEquals; + import java.net.URL; -import java.util.Collection; +import java.util.*; public class TestHiveFunctionLoader { private TajoConf conf = new TajoConf(); @@ -63,6 +62,16 @@ public final void tearDown() { @Test public void testAnalyzeUDFclass() { - HiveFunctionLoader.analyzeUDFclasses(); + Set> funcSet = new HashSet<>(); + funcSet.add(HiveUDFtest.class); + List funcList = new LinkedList<>(); + + HiveFunctionLoader.analyzeUDFclasses(funcSet, funcList); + + assertEquals(funcList.size(), 1); + + FunctionDesc desc = funcList.get(0); + + assertEquals(desc.getFunctionName(), "multiplestr"); } } \ No newline at end of file From e2c0d719fa5c707fa08775b756aff45eb7cd19c6 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 9 Nov 2015 15:03:31 +0900 Subject: [PATCH 15/72] License added --- .../engine/function/hiveudf/HiveUDFtest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java index 996db84f4e..72d88a8d14 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java @@ -1,3 +1,21 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.tajo.engine.function.hiveudf; import org.apache.hadoop.hive.ql.exec.Description; From e675e15067fbf1effdafb3bfa0fc76da895f6a36 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 9 Nov 2015 15:12:19 +0900 Subject: [PATCH 16/72] deterministic added and test modified --- .../org/apache/tajo/catalog/FunctionDesc.java | 4 ++ .../tajo/function/FunctionSignature.java | 14 +++++++ .../src/main/proto/CatalogProtos.proto | 1 + .../function/hiveudf/HiveFunctionLoader.java | 20 ++++++---- .../hiveudf/HiveGeneralFunctionHolder.java | 38 +++++++++++++++++++ 5 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveGeneralFunctionHolder.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java index d1306f9717..76d4f244f7 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java @@ -116,6 +116,10 @@ public DataType getReturnType() { return signature.getReturnType(); } + public boolean isDeterministic() { + return signature.isDeterministic(); + } + //////////////////////////////////////// // Invocation //////////////////////////////////////// diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java index 89ee01713a..aa402511e5 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java @@ -39,6 +39,8 @@ public class FunctionSignature implements Comparable, ProtoOb private DataType[] paramTypes; @Expose private DataType returnType; + @Expose + private boolean deterministic = true; public FunctionSignature(FunctionType type, String name, DataType returnType, @NotNull DataType... params) { this.functionType = type; @@ -52,6 +54,14 @@ public FunctionSignature(FunctionSignatureProto proto) { this.name = proto.getName(); this.paramTypes = proto.getParameterTypesList().toArray(new DataType[proto.getParameterTypesCount()]); this.returnType = proto.getReturnType(); + if (proto.hasDeterministic()) { + this.deterministic = proto.getDeterministic(); + } + } + + public FunctionSignature(FunctionType type, String name, DataType returnType, boolean deterministic, @NotNull DataType... params) { + this(type, name, returnType, params); + this.deterministic = deterministic; } public FunctionType getFunctionType() { @@ -70,6 +80,10 @@ public DataType getReturnType() { return returnType; } + public boolean isDeterministic() { + return deterministic; + } + @Override public int hashCode() { return Objects.hashCode(functionType, name, returnType, Objects.hashCode(paramTypes)); diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 2b3409570d..773e474243 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -359,6 +359,7 @@ message FunctionSignatureProto { required string name = 2; required DataType return_type = 3; repeated DataType parameter_types = 4; + optional bool deterministic = 5; } message FunctionSupplementProto { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index bf188fac48..ddcea92ea4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.io.*; import org.apache.tajo.catalog.CatalogUtil; @@ -30,6 +31,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.function.*; import org.reflections.Reflections; import org.reflections.util.ConfigurationBuilder; @@ -125,15 +127,19 @@ static void analyzeUDFclasses(Set> classes, List(HiveGeneralFunctionHolder.class)); + FunctionSupplement supplement = new FunctionSupplement(); + + FunctionDesc funcDesc = new FunctionDesc(signature, invocation, supplement); -// UDFType type = clazz.getDeclaredAnnotation(UDFType.class); -// if (type != null) { -// deterministic = type.deterministic(); -// } + list.add(funcDesc); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveGeneralFunctionHolder.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveGeneralFunctionHolder.java new file mode 100644 index 0000000000..36b9a900c7 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveGeneralFunctionHolder.java @@ -0,0 +1,38 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.hiveudf; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.plan.function.GeneralFunction; +import org.apache.tajo.storage.Tuple; + +/** + * Class to wrap Hive Scalar UDF + */ +public class HiveGeneralFunctionHolder extends GeneralFunction { + public HiveGeneralFunctionHolder(Column[] definedArgs) { + super(definedArgs); + } + + @Override + public Datum eval(Tuple params) { + return null; + } +} From 6df6409852fd324c9d83b02fbd0d077f3ee08d27 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 9 Nov 2015 17:40:09 +0900 Subject: [PATCH 17/72] deterministic test --- .../org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java | 3 +++ .../tajo/engine/function/hiveudf/TestHiveFunctionLoader.java | 1 + 2 files changed, 4 insertions(+) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java index 72d88a8d14..d608e7642b 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java @@ -20,9 +20,12 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; +// Actually it's deterministic, but it is set as false to test +@UDFType(deterministic = false) @Description(name="multiplestr") public class HiveUDFtest extends UDF { Text evaluate(Text str, IntWritable num) { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 280196235f..399c70526a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -73,5 +73,6 @@ public void testAnalyzeUDFclass() { FunctionDesc desc = funcList.get(0); assertEquals(desc.getFunctionName(), "multiplestr"); + assertEquals(desc.isDeterministic(), false); } } \ No newline at end of file From 110c5882eba1780e323412c4339b11154ad62ea8 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 10 Nov 2015 16:59:37 +0900 Subject: [PATCH 18/72] More test and FunctionDescBuilder added --- .../tajo/catalog/FunctionDescBuilder.java | 72 +++++++++++++++++++ .../engine/function/hiveudf/HiveUDFtest.java | 13 +++- .../hiveudf/TestHiveFunctionLoader.java | 4 ++ .../function/hiveudf/HiveFunctionLoader.java | 15 ++-- 4 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java new file mode 100644 index 0000000000..da16f63a90 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java @@ -0,0 +1,72 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.function.*; + +public class FunctionDescBuilder { + private String name = null; + private FunctionType funcType = null; + private DataType retType = null; + private DataType [] params = null; + private Class clazz = null; + private boolean isDeterministic = true; + + public FunctionDescBuilder setName(String name) { + this.name = name; + return this; + } + + public FunctionDescBuilder setFunctionType(FunctionType type) { + funcType = type; + return this; + } + + public FunctionDescBuilder setReturnType(DataType type) { + retType = type; + return this; + } + + public FunctionDescBuilder setParams(DataType [] params) { + this.params = params; + return this; + } + + public FunctionDescBuilder setClass(Class clazz) { + this.clazz = clazz; + return this; + } + + public FunctionDescBuilder setDeterministic(boolean isDeterministic) { + this.isDeterministic = isDeterministic; + return this; + } + + public FunctionDesc build() { + FunctionInvocation invocation = new FunctionInvocation(); + invocation.setLegacy(new ClassBaseInvocationDesc<>(clazz)); + FunctionSupplement supplement = new FunctionSupplement(); + + FunctionSignature signature = new FunctionSignature(funcType, name, retType, isDeterministic, params); + + return new FunctionDesc(signature, invocation, supplement); + } +} diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java index d608e7642b..590d8ec8d0 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/HiveUDFtest.java @@ -24,11 +24,18 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; -// Actually it's deterministic, but it is set as false to test +/** + * Hive UDF sample class for testing. + * Actually it's deterministic, but it is set as false to test. + */ @UDFType(deterministic = false) -@Description(name="multiplestr") +@Description( + name="multiplestr", + value = "repeat string", + extended = "multiplestr(str, num)" +) public class HiveUDFtest extends UDF { - Text evaluate(Text str, IntWritable num) { + public Text evaluate(Text str, IntWritable num) { String origin = str.toString(); for (int i=0; i Set> getSubclassesFromJarEntry(URL[] urls, static void analyzeUDFclasses(Set> classes, List list) { for (Class clazz: classes) { String [] names; - boolean deterministic = true; Description desc = clazz.getAnnotation(Description.class); if (desc != null) { @@ -127,19 +127,16 @@ static void analyzeUDFclasses(Set> classes, List(HiveGeneralFunctionHolder.class)); - FunctionSupplement supplement = new FunctionSupplement(); - - FunctionDesc funcDesc = new FunctionDesc(signature, invocation, supplement); + builder.setName(names[0]).setFunctionType(CatalogProtos.FunctionType.UDF).setReturnType(retType).setParams(params); - list.add(funcDesc); + list.add(builder.build()); } } From 2142e3a0dca128c64c81dc22c8fe437ac60f4167 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Wed, 11 Nov 2015 18:28:12 +0900 Subject: [PATCH 19/72] test refactoring --- .../hiveudf/TestHiveFunctionLoader.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 4ffe9f1247..2cc5bfcb82 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -36,24 +36,22 @@ import java.util.*; public class TestHiveFunctionLoader { - private TajoConf conf = new TajoConf(); - private CatalogService catService; private TajoTestingCluster cluster; @Before public final void setUp() throws Exception { - LocalTajoTestingUtility util = new LocalTajoTestingUtility(); cluster = new TajoTestingCluster(); - cluster.startCatalogCluster(); - catService = cluster.getCatalogService(); + + TajoConf conf = cluster.getConfiguration(); URL hiveUDFURL = ClassLoader.getSystemResource("hiveudf"); Preconditions.checkNotNull(hiveUDFURL, "hive udf directory is absent."); - conf.set("hive.udf.dir", hiveUDFURL.toString().substring("file:".length())); - HiveFunctionLoader.loadHiveUDFs(conf); + cluster.startMiniClusterInLocal(1); + cluster.startCatalogCluster(); + catService = cluster.getCatalogService(); } @After @@ -73,10 +71,17 @@ public void testAnalyzeUDFclass() { FunctionDesc desc = funcList.get(0); - assertEquals(desc.getFunctionName(), "multiplestr"); - assertEquals(desc.isDeterministic(), false); - assertEquals(desc.getReturnType().getType(), TajoDataTypes.Type.TEXT); - assertEquals(desc.getParamTypes()[0].getType(), TajoDataTypes.Type.TEXT); - assertEquals(desc.getParamTypes()[1].getType(), TajoDataTypes.Type.INT4); + assertEquals("multiplestr", desc.getFunctionName()); + assertEquals(false, desc.isDeterministic()); + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); + assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[1].getType()); + } + + @Test + public void testFindFunction() throws Exception { + Collection descs = catService.getFunctions(); + + assertEquals(2, descs.size()); } } \ No newline at end of file From 5fd2c9187ac137fd34f0e03d8bf59d8db9cc2ef9 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Wed, 11 Nov 2015 18:30:04 +0900 Subject: [PATCH 20/72] setup sequence refined --- .../apache/tajo/engine/eval/ExprTestBase.java | 6 ++++-- .../hiveudf/TestHiveFunctionLoader.java | 2 -- .../tajo/engine/function/FunctionLoader.java | 16 +++++++-------- .../function/hiveudf/HiveFunctionLoader.java | 7 ++----- .../hiveudf/HiveGeneralFunctionHolder.java | 4 ++-- .../org/apache/tajo/master/TajoMaster.java | 20 +++++++++---------- .../org/apache/tajo/worker/TajoWorker.java | 5 +++-- 7 files changed, 28 insertions(+), 32 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index c416b8e349..ffbf7255f3 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -61,6 +61,7 @@ import org.junit.BeforeClass; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -95,8 +96,9 @@ public static void setUp() throws Exception { cat.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234/warehouse"); cat.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); Map map = FunctionLoader.load(); - map = FunctionLoader.loadUserDefinedFunctions(conf, map); - for (FunctionDesc funcDesc : map.values()) { + List list = new ArrayList<>(map.values()); + list = FunctionLoader.loadUserDefinedFunctions(conf, list); + for (FunctionDesc funcDesc : list) { cat.createFunction(funcDesc); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 2cc5bfcb82..19a477c083 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -81,7 +81,5 @@ public void testAnalyzeUDFclass() { @Test public void testFindFunction() throws Exception { Collection descs = catService.getFunctions(); - - assertEquals(2, descs.size()); } } \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java index 2284d52e5d..411942afae 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java @@ -36,6 +36,7 @@ import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamOptionTypes; import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.engine.function.hiveudf.HiveGeneralFunctionHolder; import org.apache.tajo.function.*; import org.apache.tajo.plan.function.python.PythonScriptEngine; import org.apache.tajo.util.ClassUtil; @@ -86,12 +87,11 @@ public static Map load() { * Load functions defined by users. * * @param conf - * @param functionMap + * @param functionList * @return * @throws IOException */ - public static Map loadUserDefinedFunctions(TajoConf conf, - Map functionMap) + public static List loadUserDefinedFunctions(TajoConf conf, List functionList) throws IOException { String[] codePaths = conf.getStrings(TajoConf.ConfVars.PYTHON_CODE_DIR.varname); @@ -117,14 +117,12 @@ public static Map loadUserDefinedFunctions(Tajo filePaths.add(codePath); } for (Path filePath : filePaths) { - for (FunctionDesc f : PythonScriptEngine.registerFunctions(filePath.toUri(), - FunctionLoader.PYTHON_FUNCTION_NAMESPACE)) { - functionMap.put(f.getSignature(), f); - } + PythonScriptEngine.registerFunctions(filePath.toUri(), FunctionLoader.PYTHON_FUNCTION_NAMESPACE) + .stream().map(functionList::add); } } } - return functionMap; + return functionList; } public static Set findScalarFunctions() { @@ -220,7 +218,7 @@ public static List findLegacyFunctions() { Set functionClasses = ClassUtil.findClasses(Function.class, "org.apache.tajo.engine.function"); for (Class eachClass : functionClasses) { - if(eachClass.isInterface() || Modifier.isAbstract(eachClass.getModifiers())) { + if(eachClass.isInterface() || Modifier.isAbstract(eachClass.getModifiers()) || eachClass == HiveGeneralFunctionHolder.class) { continue; } Function function = null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index aed8944890..13eee8d124 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -102,10 +102,6 @@ static void analyzeUDFclasses(Set> classes, List> classes, List loadFunctions() throws IOException { - Map functionMap = FunctionLoader.load(); - return FunctionLoader.loadUserDefinedFunctions(systemConf, functionMap).values(); + List functionList = new ArrayList<>(FunctionLoader.load().values()); + Collection funcs = FunctionLoader.loadUserDefinedFunctions(systemConf, functionList); + funcs.addAll(HiveFunctionLoader.loadHiveUDFs(systemConf)); + + return funcs; } private void initSystemMetrics() { systemMetrics = new TajoSystemMetrics(systemConf, Master.class, getMasterName()); systemMetrics.start(); - systemMetrics.register(Master.Cluster.UPTIME, new Gauge() { - @Override - public Long getValue() { - return context.getClusterUptime(); - } - }); - + systemMetrics.register(Master.Cluster.UPTIME, () -> context.getClusterUptime()); systemMetrics.register(Master.Cluster.class, new ClusterResourceMetricSet(context)); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index 8315c1ca80..a516b64a9b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -34,6 +34,7 @@ import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.function.FunctionLoader; +import org.apache.tajo.engine.function.hiveudf.HiveFunctionLoader; import org.apache.tajo.master.cluster.WorkerConnectionInfo; import org.apache.tajo.metrics.Node; import org.apache.tajo.plan.function.python.PythonScriptEngine; @@ -199,7 +200,8 @@ public void serviceInit(Configuration conf) throws Exception { historyReader = new HistoryReader(workerContext.getWorkerName(), this.systemConf); - FunctionLoader.loadUserDefinedFunctions(systemConf, new HashMap<>()); + FunctionLoader.loadUserDefinedFunctions(systemConf, new ArrayList<>()); + HiveFunctionLoader.loadHiveUDFs(systemConf); PythonScriptEngine.initPythonScriptEngineFiles(); @@ -208,7 +210,6 @@ public void serviceInit(Configuration conf) throws Exception { private void initWorkerMetrics() { workerSystemMetrics = new TajoSystemMetrics(systemConf, Node.class, workerContext.getWorkerName()); - workerSystemMetrics.start(); workerSystemMetrics.register(Node.QueryMaster.RUNNING_QM, From b4a7015d485e253195c29657373519aad12880fd Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 12 Nov 2015 17:57:40 +0900 Subject: [PATCH 21/72] Method names are changed --- .../tajo/engine/function/hiveudf/HiveFunctionLoader.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index 13eee8d124..6cdbce8045 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -32,7 +32,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.function.*; import org.reflections.Reflections; import org.reflections.util.ConfigurationBuilder; @@ -62,13 +61,13 @@ public static Collection loadHiveUDFs(TajoConf conf) { // For UDF's decendants (legacy) Set> udfClasses = getSubclassesFromJarEntry(urls, UDF.class); if (udfClasses != null) { - analyzeUDFclasses(udfClasses, funcList); + extractUDFclasses(udfClasses, funcList); } // For GenericUDF's decendants (newer interface) Set> genericUDFclasses = getSubclassesFromJarEntry(urls, GenericUDF.class); if (genericUDFclasses != null) { - analyzeGenericUDFclasses(genericUDFclasses, funcList); + extractGenericUDFclasses(genericUDFclasses, funcList); } } @@ -87,7 +86,7 @@ private static Set> getSubclassesFromJarEntry(URL[] urls, return refl.getSubTypesOf(targetCls); } - static void analyzeUDFclasses(Set> classes, List list) { + static void extractUDFclasses(Set> classes, List list) { for (Class clazz: classes) { String [] names; @@ -137,7 +136,7 @@ static void analyzeUDFclasses(Set> classes, List> classes, ArrayList list) { + private static void extractGenericUDFclasses(Set> classes, ArrayList list) { } private static TajoDataTypes.DataType convertHiveTypeToTajoType(Class hiveType) { From 96a8c72a5f59079a8f782b861f970bf77e588e97 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 12 Nov 2015 17:59:33 +0900 Subject: [PATCH 22/72] Test problem fixed --- .../function/hiveudf/TestHiveFunctionLoader.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 19a477c083..1e7992fc27 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -20,7 +20,6 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.FunctionDesc; @@ -34,9 +33,10 @@ import java.net.URL; import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class TestHiveFunctionLoader { - private CatalogService catService; private TajoTestingCluster cluster; @Before @@ -49,9 +49,7 @@ public final void setUp() throws Exception { Preconditions.checkNotNull(hiveUDFURL, "hive udf directory is absent."); conf.set("hive.udf.dir", hiveUDFURL.toString().substring("file:".length())); - cluster.startMiniClusterInLocal(1); - cluster.startCatalogCluster(); - catService = cluster.getCatalogService(); + cluster.startMiniClusterInLocal(0); } @After @@ -65,7 +63,7 @@ public void testAnalyzeUDFclass() { funcSet.add(HiveUDFtest.class); List funcList = new LinkedList<>(); - HiveFunctionLoader.analyzeUDFclasses(funcSet, funcList); + HiveFunctionLoader.extractUDFclasses(funcSet, funcList); assertEquals(funcList.size(), 1); @@ -80,6 +78,7 @@ public void testAnalyzeUDFclass() { @Test public void testFindFunction() throws Exception { + CatalogService catService = cluster.getMaster().getCatalog(); Collection descs = catService.getFunctions(); } } \ No newline at end of file From 96b9294495d27cb098147765db7841816273aa27 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 16 Nov 2015 12:28:52 +0900 Subject: [PATCH 23/72] AmbiguousFunctionException in loading function --- .../org/apache/tajo/catalog/FunctionDesc.java | 11 ++++++ .../tajo/function/FunctionSignature.java | 17 +++++++++ .../apache/tajo/engine/eval/ExprTestBase.java | 2 +- .../tajo/engine/function/FunctionLoader.java | 3 +- .../org/apache/tajo/master/TajoMaster.java | 35 +++++++++++++------ .../org/apache/tajo/worker/TajoWorker.java | 2 +- 6 files changed, 57 insertions(+), 13 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java index 76d4f244f7..b2c96149e3 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java @@ -161,6 +161,10 @@ public void setExample(String example) { public int hashCode() { return Objects.hashCode(signature); } + + public int hashCodeWithoutType() { + return signature.hashCodeWithoutType(); + } @Override public boolean equals(Object obj) { @@ -171,6 +175,13 @@ public boolean equals(Object obj) { } return false; } + + public boolean equalsSignature(Object obj) { + if (obj instanceof FunctionDesc) { + return this.getSignature().equalsWithoutType(obj); + } + return false; + } @Override public Object clone() throws CloneNotSupportedException{ diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java index aa402511e5..6d634956fd 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java @@ -158,4 +158,21 @@ public int compareTo(FunctionSignature o) { return o.paramTypes.length - paramTypes.length; } + + public boolean equalsWithoutType(Object obj) { + if (obj instanceof FunctionSignature) { + FunctionSignature other = (FunctionSignature) obj; + + boolean eq = name.equals(other.name); + eq = eq && TUtil.checkEquals(paramTypes, other.paramTypes); + eq = eq && returnType.equals(other.returnType); + return eq; + } else { + return false; + } + } + + public int hashCodeWithoutType() { + return Objects.hashCode(name, returnType, Objects.hashCode(paramTypes)); + } } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index ffbf7255f3..a75054c655 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -97,7 +97,7 @@ public static void setUp() throws Exception { cat.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); Map map = FunctionLoader.load(); List list = new ArrayList<>(map.values()); - list = FunctionLoader.loadUserDefinedFunctions(conf, list); + list.addAll(FunctionLoader.loadUserDefinedFunctions(conf)); for (FunctionDesc funcDesc : list) { cat.createFunction(funcDesc); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java index 411942afae..8d33ea45bc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java @@ -91,8 +91,9 @@ public static Map load() { * @return * @throws IOException */ - public static List loadUserDefinedFunctions(TajoConf conf, List functionList) + public static List loadUserDefinedFunctions(TajoConf conf) throws IOException { + List functionList = new LinkedList<>(); String[] codePaths = conf.getStrings(TajoConf.ConfVars.PYTHON_CODE_DIR.varname); if (codePaths != null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index 1bb006772d..cf864c6c1b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -19,7 +19,6 @@ package org.apache.tajo.master; import com.google.common.annotations.VisibleForTesting; -import org.apache.avro.generic.GenericData; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -48,7 +47,6 @@ import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.function.hiveudf.HiveFunctionLoader; import org.apache.tajo.exception.*; -import org.apache.tajo.function.FunctionSignature; import org.apache.tajo.master.rm.TajoResourceManager; import org.apache.tajo.metrics.ClusterResourceMetricSet; import org.apache.tajo.metrics.Master; @@ -77,10 +75,7 @@ import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; @@ -218,12 +213,32 @@ public void serviceInit(Configuration conf) throws Exception { LOG.info("Tajo Master is initialized."); } - private Collection loadFunctions() throws IOException { + private Collection loadFunctions() throws IOException, AmbiguousFunctionException { List functionList = new ArrayList<>(FunctionLoader.load().values()); - Collection funcs = FunctionLoader.loadUserDefinedFunctions(systemConf, functionList); - funcs.addAll(HiveFunctionLoader.loadHiveUDFs(systemConf)); + Collection udfs = FunctionLoader.loadUserDefinedFunctions(systemConf); + Collection hiveUDFs = HiveFunctionLoader.loadHiveUDFs(systemConf); - return funcs; + HashMap funcSet = new HashMap<>(); + + for (FunctionDesc desc: functionList) { + funcSet.put(desc.hashCodeWithoutType(), desc); + } + + checkUDFduplicates(udfs, funcSet); + checkUDFduplicates(hiveUDFs, funcSet); + + return funcSet.values(); + } + + private void checkUDFduplicates(Collection udfs, HashMap funcSet) + throws AmbiguousFunctionException { + for (FunctionDesc desc: udfs) { + if (funcSet.containsKey(desc.hashCodeWithoutType())) { + throw new AmbiguousFunctionException(String.format("UDF %s", desc.toString())); + } + + funcSet.put(desc.hashCodeWithoutType(), desc); + } } private void initSystemMetrics() { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index a516b64a9b..53a22f8a16 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -200,7 +200,7 @@ public void serviceInit(Configuration conf) throws Exception { historyReader = new HistoryReader(workerContext.getWorkerName(), this.systemConf); - FunctionLoader.loadUserDefinedFunctions(systemConf, new ArrayList<>()); + FunctionLoader.loadUserDefinedFunctions(systemConf); HiveFunctionLoader.loadHiveUDFs(systemConf); PythonScriptEngine.initPythonScriptEngineFiles(); From 2b43dc1dd8fff8c6aee0d4badcaae4d10abfc345 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 16 Nov 2015 14:36:56 +0900 Subject: [PATCH 24/72] Basic finding test --- .../hiveudf/TestHiveFunctionLoader.java | 13 ++++++++++++- .../src/test/resources/hiveudf/myudf.jar | Bin 1616 -> 1626 bytes 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 1e7992fc27..f85bec6f19 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; @@ -33,7 +34,6 @@ import java.net.URL; import java.util.*; -import java.util.stream.Collectors; import java.util.stream.Stream; public class TestHiveFunctionLoader { @@ -80,5 +80,16 @@ public void testAnalyzeUDFclass() { public void testFindFunction() throws Exception { CatalogService catService = cluster.getMaster().getCatalog(); Collection descs = catService.getFunctions(); + + Stream upper = descs.stream().filter((FunctionDesc desc)->desc.getFunctionName().equals("my_upper")); + + FunctionDesc desc = upper.findFirst().get(); + + assertEquals(HiveGeneralFunctionHolder.class, desc.getLegacyFuncClass()); + assertEquals(CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), desc.getReturnType()); + assertEquals(1, desc.getParamTypes().length); + assertEquals(CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), desc.getParamTypes()[0]); + + upper.close(); } } \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/hiveudf/myudf.jar b/tajo-core-tests/src/test/resources/hiveudf/myudf.jar index 1afb298f33ef0ccef627ce5de61c5e3fa5869b98..b462126ac41e89476e74fb3c9f09eebeff58b072 100644 GIT binary patch delta 644 zcmcb>bBl*Jz?+$ci-CcIf#GLf!9-qVW)Nkp4yIytz*LU`h}yXSI3t+5Ut_X98+Uzc zP_)0WqlE2cZ9P_PaZyK>kUOl8j+hJgbT%gVaPe<3`FN7&_D0#wyEnf(XZAGyA-Bq` zA6bw3jp~eZYd!>(Pu@Al^1kK$nP$bhBh&Zm? zc8hze#^*^3->}EFooamN**05u>WRrpcGDx>>*u()=xiy`ShViMl)L^1x+GW3N!%;? zYR2h_9hW^mHA;EKeYyAfarb7EsW>It?* zrq1ALPvgX^uR_|cUvm{ZrJOoD+vm^xIsxZ{8;YV-+>UNqUF3Rpzn4&MOQ4nF)xG;y z?2?dI?oxHGuhaN!5WI5fWM5T5LrI-u;_ta$Z%9wcJLcWCsyFLx@4bQmjiSu;2`MR; zCfl}Ivj2X2?&a*s*BzOScR0TNYaGtpzcKdsWVV`|=&}PF%-$Z7-1blS`I`?NRz|yW zji!E)oc3oyYJR7?(#y|=UY4`>C-ZD}v+idW&${`4v(AeoVbl7ml5g6LH!Do4wmdd5 zp0(QgdcyoWA9Z?7+09;TU#y(K_u1sKvrBPMO^?yX#;14lS8O`1ud(;;UZpLs^O#B!5s#jFUmJgpCeto+mP8Ra7w!Buy6yVLy;k$UX z_A^EX23KYVh5&CyCN2g}c%qrSoy8cGFeblcF$dG?tTtdejnx56uVnS*1g#icvsBs@P delta 634 zcmcb`bAg99z?+$ci-CcIf#FGO%tT&gW)Nkp4yIytz*LU`h}yXSI3t+5Ut_W*8+Uzc zP^`a*p@{8We;KwI!4Q^-`h{#gz4>Yq!psL}@YrlPt!6jv=B{XCYuR+G*IlQi2P0nQFvxQCVa@RH{hR^oRG2M9f zLdC+M)HtJ%<_)IhYqOZjFAF*+trG9am z@SfsW6TvO3qazefa^>xw Date: Mon, 16 Nov 2015 15:08:03 +0900 Subject: [PATCH 25/72] process synonyms --- .../hiveudf/TestHiveFunctionLoader.java | 19 ++++++++++++------- .../function/hiveudf/HiveFunctionLoader.java | 7 +++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index f85bec6f19..642af62982 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -24,6 +24,7 @@ import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.junit.After; @@ -79,17 +80,21 @@ public void testAnalyzeUDFclass() { @Test public void testFindFunction() throws Exception { CatalogService catService = cluster.getMaster().getCatalog(); - Collection descs = catService.getFunctions(); - Stream upper = descs.stream().filter((FunctionDesc desc)->desc.getFunctionName().equals("my_upper")); - - FunctionDesc desc = upper.findFirst().get(); + FunctionDesc desc = catService.getFunction("my_upper", CatalogProtos.FunctionType.UDF, + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); assertEquals(HiveGeneralFunctionHolder.class, desc.getLegacyFuncClass()); - assertEquals(CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), desc.getReturnType()); + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(1, desc.getParamTypes().length); - assertEquals(CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), desc.getParamTypes()[0]); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); + + desc = catService.getFunction("test_upper", CatalogProtos.FunctionType.UDF, + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); - upper.close(); + assertEquals(HiveGeneralFunctionHolder.class, desc.getLegacyFuncClass()); + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); + assertEquals(1, desc.getParamTypes().length); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); } } \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index 6cdbce8045..de6e28f06f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -129,10 +129,13 @@ static void extractUDFclasses(Set> classes, List Date: Mon, 16 Nov 2015 15:28:24 +0900 Subject: [PATCH 26/72] process description and example --- .../tajo/catalog/FunctionDescBuilder.java | 20 +++++++++++++++++++ .../hiveudf/TestHiveFunctionLoader.java | 2 ++ .../function/hiveudf/HiveFunctionLoader.java | 12 +++++++++++ 3 files changed, 34 insertions(+) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java index da16f63a90..9c8dd4ea1e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java @@ -29,6 +29,8 @@ public class FunctionDescBuilder { private DataType [] params = null; private Class clazz = null; private boolean isDeterministic = true; + private String description = null; + private String example = null; public FunctionDescBuilder setName(String name) { this.name = name; @@ -60,11 +62,29 @@ public FunctionDescBuilder setDeterministic(boolean isDeterministic) { return this; } + public FunctionDescBuilder setDescription(String desc) { + this.description = desc; + return this; + } + + public FunctionDescBuilder setExample(String ex) { + this.example = ex; + return this; + } + public FunctionDesc build() { FunctionInvocation invocation = new FunctionInvocation(); invocation.setLegacy(new ClassBaseInvocationDesc<>(clazz)); FunctionSupplement supplement = new FunctionSupplement(); + if (description != null) { + supplement.setShortDescription(description); + } + + if (example != null) { + supplement.setExample(example); + } + FunctionSignature signature = new FunctionSignature(funcType, name, retType, isDeterministic, params); return new FunctionDesc(signature, invocation, supplement); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 642af62982..293dff5886 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -88,6 +88,7 @@ public void testFindFunction() throws Exception { assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); + assertEquals("to uppercase", desc.getDescription()); desc = catService.getFunction("test_upper", CatalogProtos.FunctionType.UDF, CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); @@ -96,5 +97,6 @@ public void testFindFunction() throws Exception { assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); + assertEquals("to uppercase", desc.getDescription()); } } \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index de6e28f06f..426ba98324 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -89,6 +89,7 @@ private static Set> getSubclassesFromJarEntry(URL[] urls, static void extractUDFclasses(Set> classes, List list) { for (Class clazz: classes) { String [] names; + String value = null, extended = null; Description desc = clazz.getAnnotation(Description.class); if (desc != null) { @@ -96,6 +97,9 @@ static void extractUDFclasses(Set> classes, List> classes, List Date: Mon, 16 Nov 2015 15:31:53 +0900 Subject: [PATCH 27/72] Non @Description UDF test added --- .../engine/function/hiveudf/TestHiveFunctionLoader.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 293dff5886..4d0d657c60 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -98,5 +98,14 @@ public void testFindFunction() throws Exception { assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); assertEquals("to uppercase", desc.getDescription()); + + // Test for UDF without @Description + desc = catService.getFunction("com_example_hive_udf_MyLower", CatalogProtos.FunctionType.UDF, + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); + + assertEquals(HiveGeneralFunctionHolder.class, desc.getLegacyFuncClass()); + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); + assertEquals(1, desc.getParamTypes().length); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); } } \ No newline at end of file From 58172a6376622e7aef9dae2333f811b150214c1e Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 1 Dec 2015 11:59:28 +0900 Subject: [PATCH 28/72] Change PythonUDF PB to UDF one --- .../tajo/function/FunctionInvocation.java | 31 +++++++------------ ...cationDesc.java => UDFInvocationDesc.java} | 28 ++++++++++------- .../src/main/proto/CatalogProtos.proto | 14 ++++++--- .../function/python/PythonScriptEngine.java | 16 +++++----- 4 files changed, 47 insertions(+), 42 deletions(-) rename tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/{PythonInvocationDesc.java => UDFInvocationDesc.java} (76%) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java index 09b056b902..4c6f4502ff 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.ProtoObject; import static org.apache.tajo.catalog.proto.CatalogProtos.FunctionInvocationProto; @@ -36,7 +37,7 @@ public class FunctionInvocation implements ProtoObject @Expose ClassBaseInvocationDesc aggregationJIT; @Expose - PythonInvocationDesc python; + UDFInvocationDesc udf; public FunctionInvocation() { } @@ -57,8 +58,8 @@ public FunctionInvocation(FunctionInvocationProto proto) { if (proto.hasAggregationJit()) { this.aggregationJIT = new ClassBaseInvocationDesc(proto.getAggregation()); } - if (proto.hasPython()) { - this.python = new PythonInvocationDesc(proto.getPython()); + if (proto.hasUdfInvocation()) { + this.udf = new UDFInvocationDesc(proto.getUdfInvocation()); } } @@ -127,27 +128,19 @@ public ClassBaseInvocationDesc getAggregationJIT() { } public boolean hasPython() { - return python != null && python.isScalarFunction(); + return udf != null && udf.getType() == CatalogProtos.UDFtype.PYTHON && udf.isScalarFunction(); } - public void setPython(PythonInvocationDesc python) { - this.python = python; + public void setUDF(UDFInvocationDesc udf) { + this.udf = udf; } - public PythonInvocationDesc getPython() { - return python; + public UDFInvocationDesc getUDF() { + return udf; } public boolean hasPythonAggregation() { - return python != null && !python.isScalarFunction(); - } - - public void setPythonAggregation(PythonInvocationDesc pythonAggregation) { - this.python = pythonAggregation; - } - - public PythonInvocationDesc getPythonAggregation() { - return this.python; + return udf != null && udf.getType() == CatalogProtos.UDFtype.PYTHON && !udf.isScalarFunction(); } @Override @@ -169,14 +162,14 @@ public FunctionInvocationProto getProto() { builder.setAggregationJit(aggregationJIT.getProto()); } if (hasPython() || hasPythonAggregation()) { - builder.setPython(python.getProto()); + builder.setUdfInvocation(udf.getProto()); } return builder.build(); } @Override public int hashCode() { - return Objects.hashCode(legacy, scalar, scalarJIT, python); + return Objects.hashCode(legacy, scalar, scalarJIT, udf); } public String toString() { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/PythonInvocationDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java similarity index 76% rename from tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/PythonInvocationDesc.java rename to tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java index d3365e5e98..f7ef63bb9d 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/PythonInvocationDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java @@ -20,27 +20,29 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; -import org.apache.tajo.catalog.proto.CatalogProtos.PythonInvocationDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.UDFtype; +import org.apache.tajo.catalog.proto.CatalogProtos.UDFinvocationDescProto; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.util.TUtil; /** - * PythonInvocationDesc describes a function name + * UDFInvocationDesc describes a function name * and a file path to the script where the function is defined. */ -public class PythonInvocationDesc implements ProtoObject, Cloneable { +public class UDFInvocationDesc implements ProtoObject, Cloneable { + @Expose private UDFtype type; @Expose private boolean isScalarFunction; // true if udf, false if udaf @Expose private String funcOrClassName; // function name if udf, class name if udaf @Expose private String filePath; // file path to the python module /** - * Constructor of {@link PythonInvocationDesc}. + * Constructor of {@link UDFInvocationDesc}. * * @param funcOrClassName if udf, function name. else, class name. * @param filePath path to script file * @param isScalarFunction */ - public PythonInvocationDesc(String funcOrClassName, String filePath, boolean isScalarFunction) { + public UDFInvocationDesc(String funcOrClassName, String filePath, boolean isScalarFunction) { this.funcOrClassName = funcOrClassName; this.filePath = filePath; this.isScalarFunction = isScalarFunction; @@ -50,10 +52,14 @@ public void setFilePath(String filePath) { this.filePath = filePath; } - public PythonInvocationDesc(PythonInvocationDescProto proto) { + public UDFInvocationDesc(UDFinvocationDescProto proto) { this(proto.getFuncName(), proto.getFilePath(), proto.getIsScalarFunction()); } + public UDFtype getType() { + return type; + } + public String getName() { return funcOrClassName; } @@ -67,16 +73,16 @@ public boolean isScalarFunction() { } @Override - public PythonInvocationDescProto getProto() { - PythonInvocationDescProto.Builder builder = PythonInvocationDescProto.newBuilder(); + public UDFinvocationDescProto getProto() { + UDFinvocationDescProto.Builder builder = UDFinvocationDescProto.newBuilder(); builder.setFuncName(funcOrClassName).setFilePath(filePath).setIsScalarFunction(isScalarFunction); return builder.build(); } @Override public boolean equals(Object o) { - if (o instanceof PythonInvocationDesc) { - PythonInvocationDesc other = (PythonInvocationDesc) o; + if (o instanceof UDFInvocationDesc) { + UDFInvocationDesc other = (UDFInvocationDesc) o; return TUtil.checkEquals(funcOrClassName, other.funcOrClassName) && TUtil.checkEquals(filePath, other.filePath) && isScalarFunction == other.isScalarFunction; } @@ -95,7 +101,7 @@ public String toString() { @Override public Object clone() throws CloneNotSupportedException { - PythonInvocationDesc clone = (PythonInvocationDesc) super.clone(); + UDFInvocationDesc clone = (UDFInvocationDesc) super.clone(); clone.funcOrClassName = funcOrClassName == null ? null : funcOrClassName; clone.filePath = filePath == null ? null : filePath; clone.isScalarFunction = isScalarFunction; diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 773e474243..d064c62586 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -374,7 +374,7 @@ message FunctionInvocationProto { optional ClassBaseInvocationDescProto aggregation = 3; optional StaticMethodInvocationDescProto scalar_jit = 4; optional ClassBaseInvocationDescProto aggregation_jit = 5; - optional PythonInvocationDescProto python = 6; + optional UDFinvocationDescProto udf_invocation = 6; } message ClassBaseInvocationDescProto { @@ -388,10 +388,16 @@ message StaticMethodInvocationDescProto { repeated string param_classes = 4; } -message PythonInvocationDescProto { - required string func_name = 1; - required string file_path = 2; +enum UDFtype { + PYTHON = 0; + HIVE = 1; +} + +message UDFinvocationDescProto { + required UDFtype type = 1; + required string func_name = 2; required bool is_scalar_function = 3; + optional string file_path = 4; } message TableResponse { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java index 19a4ac2338..1600fed90b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java @@ -32,7 +32,7 @@ import org.apache.tajo.function.FunctionInvocation; import org.apache.tajo.function.FunctionSignature; import org.apache.tajo.function.FunctionSupplement; -import org.apache.tajo.function.PythonInvocationDesc; +import org.apache.tajo.function.UDFInvocationDesc; import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.plan.function.PythonAggFunctionInvoke.PythonAggFunctionContext; import org.apache.tajo.plan.function.stream.*; @@ -90,8 +90,8 @@ public static Set registerFunctions(URI path, String namespace) th TajoDataTypes.DataType returnType = getReturnTypes(scalarFuncInfo)[0]; signature = new FunctionSignature(CatalogProtos.FunctionType.UDF, scalarFuncInfo.funcName, returnType, createParamTypes(scalarFuncInfo.paramNum)); - PythonInvocationDesc invocationDesc = new PythonInvocationDesc(scalarFuncInfo.funcName, path.getPath(), true); - invocation.setPython(invocationDesc); + UDFInvocationDesc invocationDesc = new UDFInvocationDesc(scalarFuncInfo.funcName, path.getPath(), true); + invocation.setUDF(invocationDesc); functionDescs.add(new FunctionDesc(signature, invocation, supplement)); } else { AggFuncInfo aggFuncInfo = (AggFuncInfo) funcInfo; @@ -99,9 +99,9 @@ public static Set registerFunctions(URI path, String namespace) th TajoDataTypes.DataType returnType = getReturnTypes(aggFuncInfo.getFinalResultInfo)[0]; signature = new FunctionSignature(CatalogProtos.FunctionType.UDA, aggFuncInfo.funcName, returnType, createParamTypes(aggFuncInfo.evalInfo.paramNum)); - PythonInvocationDesc invocationDesc = new PythonInvocationDesc(aggFuncInfo.className, path.getPath(), false); + UDFInvocationDesc invocationDesc = new UDFInvocationDesc(aggFuncInfo.className, path.getPath(), false); - invocation.setPythonAggregation(invocationDesc); + invocation.setUDF(invocationDesc); functionDescs.add(new FunctionDesc(signature, invocation, supplement)); } } @@ -281,7 +281,7 @@ enum COMMAND_IDX { private InputStream stderr; // stderr of the process private final FunctionSignature functionSignature; - private final PythonInvocationDesc invocationDesc; + private final UDFInvocationDesc invocationDesc; private Schema inSchema; private Schema outSchema; private int[] projectionCols; @@ -297,7 +297,7 @@ public PythonScriptEngine(FunctionDesc functionDesc) { throw new IllegalStateException("Function type must be 'python'"); } functionSignature = functionDesc.getSignature(); - invocationDesc = functionDesc.getInvocation().getPython(); + invocationDesc = functionDesc.getInvocation().getUDF(); setSchema(); } @@ -306,7 +306,7 @@ public PythonScriptEngine(FunctionDesc functionDesc, boolean firstPhase, boolean throw new IllegalStateException("Function type must be 'python'"); } functionSignature = functionDesc.getSignature(); - invocationDesc = functionDesc.getInvocation().getPython(); + invocationDesc = functionDesc.getInvocation().getUDF(); this.firstPhase = firstPhase; this.lastPhase = lastPhase; setSchema(); From ee2790b82a59eccc6456c555e1474df0f5147401 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 1 Dec 2015 11:59:49 +0900 Subject: [PATCH 29/72] runFunction test added --- .../hiveudf/TestHiveFunctionLoader.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 4d0d657c60..ba58e3f910 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -25,8 +25,10 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.client.TajoClient; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.worker.TajoWorker; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -34,6 +36,7 @@ import static org.junit.Assert.assertEquals; import java.net.URL; +import java.sql.ResultSet; import java.util.*; import java.util.stream.Stream; @@ -50,7 +53,7 @@ public final void setUp() throws Exception { Preconditions.checkNotNull(hiveUDFURL, "hive udf directory is absent."); conf.set("hive.udf.dir", hiveUDFURL.toString().substring("file:".length())); - cluster.startMiniClusterInLocal(0); + cluster.startMiniClusterInLocal(1); } @After @@ -108,4 +111,17 @@ public void testFindFunction() throws Exception { assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); } + + @Test + public void testRunFunctions() throws Exception { + TajoClient client = cluster.newTajoClient(); + + ResultSet rs = client.executeQueryAndGetResult("select my_upper('abcd')"); + rs.beforeFirst(); + String result = rs.getString(1); + + assertEquals("ABCD", result); + + rs.close(); + } } \ No newline at end of file From 6fb0ee44422540b5a1456745d4f56a7f6eb71b3e Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 1 Dec 2015 15:16:04 +0900 Subject: [PATCH 30/72] Constructor modified --- .../java/org/apache/tajo/function/UDFInvocationDesc.java | 7 ++++--- .../tajo/plan/function/python/PythonScriptEngine.java | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java index f7ef63bb9d..d19d5f177a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java @@ -42,7 +42,8 @@ public class UDFInvocationDesc implements ProtoObject, C * @param filePath path to script file * @param isScalarFunction */ - public UDFInvocationDesc(String funcOrClassName, String filePath, boolean isScalarFunction) { + public UDFInvocationDesc(UDFtype type, String funcOrClassName, String filePath, boolean isScalarFunction) { + this.type = type; this.funcOrClassName = funcOrClassName; this.filePath = filePath; this.isScalarFunction = isScalarFunction; @@ -53,7 +54,7 @@ public void setFilePath(String filePath) { } public UDFInvocationDesc(UDFinvocationDescProto proto) { - this(proto.getFuncName(), proto.getFilePath(), proto.getIsScalarFunction()); + this(proto.getType(), proto.getFuncName(), proto.getFilePath(), proto.getIsScalarFunction()); } public UDFtype getType() { @@ -75,7 +76,7 @@ public boolean isScalarFunction() { @Override public UDFinvocationDescProto getProto() { UDFinvocationDescProto.Builder builder = UDFinvocationDescProto.newBuilder(); - builder.setFuncName(funcOrClassName).setFilePath(filePath).setIsScalarFunction(isScalarFunction); + builder.setType(type).setFuncName(funcOrClassName).setFilePath(filePath).setIsScalarFunction(isScalarFunction); return builder.build(); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java index 1600fed90b..2844c7c8e9 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java @@ -90,7 +90,7 @@ public static Set registerFunctions(URI path, String namespace) th TajoDataTypes.DataType returnType = getReturnTypes(scalarFuncInfo)[0]; signature = new FunctionSignature(CatalogProtos.FunctionType.UDF, scalarFuncInfo.funcName, returnType, createParamTypes(scalarFuncInfo.paramNum)); - UDFInvocationDesc invocationDesc = new UDFInvocationDesc(scalarFuncInfo.funcName, path.getPath(), true); + UDFInvocationDesc invocationDesc = new UDFInvocationDesc(CatalogProtos.UDFtype.PYTHON, scalarFuncInfo.funcName, path.getPath(), true); invocation.setUDF(invocationDesc); functionDescs.add(new FunctionDesc(signature, invocation, supplement)); } else { @@ -99,7 +99,7 @@ public static Set registerFunctions(URI path, String namespace) th TajoDataTypes.DataType returnType = getReturnTypes(aggFuncInfo.getFinalResultInfo)[0]; signature = new FunctionSignature(CatalogProtos.FunctionType.UDA, aggFuncInfo.funcName, returnType, createParamTypes(aggFuncInfo.evalInfo.paramNum)); - UDFInvocationDesc invocationDesc = new UDFInvocationDesc(aggFuncInfo.className, path.getPath(), false); + UDFInvocationDesc invocationDesc = new UDFInvocationDesc(CatalogProtos.UDFtype.PYTHON, aggFuncInfo.className, path.getPath(), false); invocation.setUDF(invocationDesc); functionDescs.add(new FunctionDesc(signature, invocation, supplement)); From ae0624a6558bf780df7fc41f59c6ff8bf9ada695 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 1 Dec 2015 17:06:25 +0900 Subject: [PATCH 31/72] toString() modified --- .../main/java/org/apache/tajo/function/UDFInvocationDesc.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java index d19d5f177a..3c56f36797 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java @@ -97,7 +97,8 @@ public int hashCode() { @Override public String toString() { - return isScalarFunction ? "[UDF] " : "[UDAF] " + funcOrClassName + " at " + filePath; + return '['+type.toString()+'/'+ (isScalarFunction ? "UDF] " : "UDAF] ") + + funcOrClassName + (filePath != null ? (" at " + filePath) : ""); } @Override From 96edfe5347523a69bf31b2d972d734faecdcbe9b Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 1 Dec 2015 17:12:46 +0900 Subject: [PATCH 32/72] UDF invocation refined --- .../apache/tajo/catalog/FunctionDescBuilder.java | 15 ++++++++++++++- .../apache/tajo/function/UDFInvocationDesc.java | 6 +++++- .../function/hiveudf/HiveFunctionLoader.java | 7 +++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java index 9c8dd4ea1e..dae97abe2d 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDescBuilder.java @@ -28,6 +28,7 @@ public class FunctionDescBuilder { private DataType retType = null; private DataType [] params = null; private Class clazz = null; + private UDFInvocationDesc udfInvocation = null; private boolean isDeterministic = true; private String description = null; private String example = null; @@ -72,9 +73,21 @@ public FunctionDescBuilder setExample(String ex) { return this; } + public FunctionDescBuilder setUDF(UDFInvocationDesc udf) { + this.udfInvocation = udf; + return this; + } + + public FunctionDesc build() { FunctionInvocation invocation = new FunctionInvocation(); - invocation.setLegacy(new ClassBaseInvocationDesc<>(clazz)); + if (funcType == FunctionType.UDF) { + invocation.setUDF(udfInvocation); + } + else { + invocation.setLegacy(new ClassBaseInvocationDesc<>(clazz)); + } + FunctionSupplement supplement = new FunctionSupplement(); if (description != null) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java index 3c56f36797..ddda9624cf 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java @@ -32,9 +32,13 @@ public class UDFInvocationDesc implements ProtoObject, Cloneable { @Expose private UDFtype type; @Expose private boolean isScalarFunction; // true if udf, false if udaf - @Expose private String funcOrClassName; // function name if udf, class name if udaf + @Expose private String funcOrClassName; // function name if python udf, class name if python udaf or others @Expose private String filePath; // file path to the python module + public UDFInvocationDesc(UDFtype type, String funcOrClassName, boolean isScalarFunction) { + this(type, funcOrClassName, null, isScalarFunction); + } + /** * Constructor of {@link UDFInvocationDesc}. * diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index 426ba98324..8a52a8d0de 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -32,6 +32,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.function.UDFInvocationDesc; import org.reflections.Reflections; import org.reflections.util.ConfigurationBuilder; @@ -133,8 +134,7 @@ static void extractUDFclasses(Set> classes, List> classes, List Date: Tue, 8 Dec 2015 17:21:57 +0900 Subject: [PATCH 33/72] Rough running test passed --- .../tajo/function/FunctionInvocation.java | 8 +- .../tajo/function/UDFInvocationDesc.java | 7 +- .../hiveudf/TestHiveFunctionLoader.java | 3 +- .../function/hiveudf/HiveFunctionLoader.java | 6 +- .../tajo/plan/function/FunctionInvoke.java | 2 + .../plan/function/HiveFunctionInvoke.java | 90 +++++++++++++++++++ 6 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java index 4c6f4502ff..835fed4827 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java @@ -131,6 +131,10 @@ public boolean hasPython() { return udf != null && udf.getType() == CatalogProtos.UDFtype.PYTHON && udf.isScalarFunction(); } + public boolean hasHiveUDF() { + return udf != null && udf.getType() == CatalogProtos.UDFtype.HIVE && udf.isScalarFunction(); + } + public void setUDF(UDFInvocationDesc udf) { this.udf = udf; } @@ -161,7 +165,7 @@ public FunctionInvocationProto getProto() { if (hasAggregationJIT()) { builder.setAggregationJit(aggregationJIT.getProto()); } - if (hasPython() || hasPythonAggregation()) { + if (hasPython() || hasPythonAggregation() || hasHiveUDF()) { builder.setUdfInvocation(udf.getProto()); } return builder.build(); @@ -175,6 +179,6 @@ public int hashCode() { public String toString() { return "legacy=" + hasLegacy() + ",scalar=" + hasScalar() + ",agg=" + hasAggregation() + ",scalarJIT=" + hasScalarJIT() + ",aggJIT=" + hasAggregationJIT() + ",python=" + hasPython() + - ",aggPython=" + hasPythonAggregation(); + ",aggPython=" + hasPythonAggregation() + ",hiveUDF="+hasHiveUDF(); } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java index ddda9624cf..c57194c615 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/UDFInvocationDesc.java @@ -33,7 +33,7 @@ public class UDFInvocationDesc implements ProtoObject, C @Expose private UDFtype type; @Expose private boolean isScalarFunction; // true if udf, false if udaf @Expose private String funcOrClassName; // function name if python udf, class name if python udaf or others - @Expose private String filePath; // file path to the python module + @Expose private String filePath; // file path to the python module, or jar path to hive udf public UDFInvocationDesc(UDFtype type, String funcOrClassName, boolean isScalarFunction) { this(type, funcOrClassName, null, isScalarFunction); @@ -80,7 +80,10 @@ public boolean isScalarFunction() { @Override public UDFinvocationDescProto getProto() { UDFinvocationDescProto.Builder builder = UDFinvocationDescProto.newBuilder(); - builder.setType(type).setFuncName(funcOrClassName).setFilePath(filePath).setIsScalarFunction(isScalarFunction); + builder.setType(type).setFuncName(funcOrClassName).setIsScalarFunction(isScalarFunction); + if (filePath != null) { + builder.setFilePath(filePath); + } return builder.build(); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index ba58e3f910..128e24ecf3 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -67,7 +67,7 @@ public void testAnalyzeUDFclass() { funcSet.add(HiveUDFtest.class); List funcList = new LinkedList<>(); - HiveFunctionLoader.extractUDFclasses(funcSet, funcList); + HiveFunctionLoader.extractUDFclasses(funcSet, funcList, null); assertEquals(funcList.size(), 1); @@ -118,6 +118,7 @@ public void testRunFunctions() throws Exception { ResultSet rs = client.executeQueryAndGetResult("select my_upper('abcd')"); rs.beforeFirst(); + rs.next(); String result = rs.getString(1); assertEquals("ABCD", result); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index 8a52a8d0de..e05a9c24ef 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -62,7 +62,7 @@ public static Collection loadHiveUDFs(TajoConf conf) { // For UDF's decendants (legacy) Set> udfClasses = getSubclassesFromJarEntry(urls, UDF.class); if (udfClasses != null) { - extractUDFclasses(udfClasses, funcList); + extractUDFclasses(udfClasses, funcList, "jar:"+urls[0].getPath()); } // For GenericUDF's decendants (newer interface) @@ -87,7 +87,7 @@ private static Set> getSubclassesFromJarEntry(URL[] urls, return refl.getSubTypesOf(targetCls); } - static void extractUDFclasses(Set> classes, List list) { + static void extractUDFclasses(Set> classes, List list, String jarurl) { for (Class clazz: classes) { String [] names; String value = null, extended = null; @@ -144,7 +144,7 @@ static void extractUDFclasses(Set> classes, List udfclass = loader.loadClass(udfDesc.getName()); + evalMethod = getEvaluateMethod(udfclass); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + + private Method getEvaluateMethod(Class clazz) { + Constructor constructor = clazz.getConstructors()[0]; + + try { + instance = constructor.newInstance(); + } catch (InstantiationException|IllegalAccessException|InvocationTargetException e) { + e.printStackTrace(); + } + + for (Method m: clazz.getMethods()) { + if (m.getName().equals("evaluate")) { + return m; + } + } + + return null; + } + + @Override + public Datum eval(Tuple tuple) { + Datum resultDatum = NullDatum.get(); + + try { + Text result = (Text)evalMethod.invoke(instance, new Text(tuple.getText(0))); + resultDatum = new TextDatum(result.toString()); + } catch (Exception e) { + e.printStackTrace(); + } + + return resultDatum; + } +} From 74a091e389c3957d482b825dce82b1cbc90d971d Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 8 Dec 2015 17:22:07 +0900 Subject: [PATCH 34/72] refactoring --- .../org/apache/tajo/plan/LogicalPlanner.java | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 6c38c02cf7..227d7a515a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -461,20 +461,16 @@ private void addNamedExprs(QueryBlock block, String [] referenceNames, ExprNorma */ private EvalExprNode buildPlanForNoneFromStatement(PlanContext context, Stack stack, Projection projection) throws TajoException { - LogicalPlan plan = context.plan; QueryBlock block = context.queryBlock; - int finalTargetNum = projection.getNamedExprs().size(); List targets = new ArrayList<>(); - for (int i = 0; i < finalTargetNum; i++) { - NamedExpr namedExpr = projection.getNamedExprs().get(i); - EvalNode evalNode = exprAnnotator.createEvalNode(context, namedExpr.getExpr(), NameResolvingMode.RELS_ONLY); - if (namedExpr.hasAlias()) { - targets.add(new Target(evalNode, namedExpr.getAlias())); - } else { - targets.add(new Target(evalNode, context.plan.generateUniqueColumnName(namedExpr.getExpr()))); - } + for (NamedExpr namedExpr: projection.getNamedExprs()) { + Expr expr = namedExpr.getExpr(); + EvalNode evalNode = exprAnnotator.createEvalNode(context, expr, NameResolvingMode.RELS_ONLY); + + String alias = namedExpr.hasAlias() ? namedExpr.getAlias() : context.plan.generateUniqueColumnName(expr); + targets.add(new Target(evalNode, alias)); } EvalExprNode evalExprNode = context.queryBlock.getNodeFromExpr(projection); evalExprNode.setTargets(targets); @@ -490,8 +486,7 @@ private List buildTargets(PlanContext context, String[] referenceNames) List targets = new ArrayList<>(); - for (int i = 0; i < referenceNames.length; i++) { - String refName = referenceNames[i]; + for (String refName: referenceNames) { if (block.isConstReference(refName)) { targets.add(new Target(block.getConstByReference(refName), refName)); } else if (block.namedExprsMgr.isEvaluated(refName)) { From 34c43bf92903922f9972051d29c97fe8150397c0 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Fri, 11 Dec 2015 16:06:39 +0900 Subject: [PATCH 35/72] type conversion added --- .../tajo/util/TajoHiveTypeConverter.java | 83 +++++++++++++++++++ .../function/hiveudf/HiveFunctionLoader.java | 28 +------ .../plan/function/HiveFunctionInvoke.java | 13 ++- 3 files changed, 96 insertions(+), 28 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java b/tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java new file mode 100644 index 0000000000..0dc1e8bce0 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.util; + +import org.apache.hadoop.io.*; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.datum.*; + +public class TajoHiveTypeConverter { + + public static DataType convertHiveTypeToTajoType(Class hiveType) { + if (hiveType == null) + return null; + + DataType.Builder builder = DataType.newBuilder(); + + if (hiveType == IntWritable.class) { + return builder.setType(Type.INT4).build(); + } + if (hiveType == LongWritable.class) { + return builder.setType(Type.INT8).build(); + } + if (hiveType == Text.class) { + return builder.setType(Type.TEXT).build(); + } + if (hiveType == FloatWritable.class) { + return builder.setType(Type.FLOAT4).build(); + } + if (hiveType == DoubleWritable.class) { + return builder.setType(Type.FLOAT8).build(); + } + + return builder.setType(Type.NULL_TYPE).build(); + } + + public static Writable convertDatum2Writable(Datum value) { + switch(value.type()) { + case INT2: return new IntWritable(value.asInt2()); + case INT4: return new IntWritable(value.asInt4()); + case INT8: return new LongWritable(value.asInt8()); + + case FLOAT4: return new FloatWritable(value.asFloat4()); + case FLOAT8: return new DoubleWritable(value.asFloat8()); + + case TEXT: return new Text(value.asChars()); + } + + return NullWritable.get(); + } + + public static Datum convertWritable2Datum(Writable value) { + DataType type = convertHiveTypeToTajoType(value.getClass()); + + switch(type.getType()) { + case INT4: return new Int4Datum(((IntWritable)value).get()); + case INT8: return new Int8Datum(((LongWritable)value).get()); + + case FLOAT4: return new Float4Datum(((FloatWritable)value).get()); + case FLOAT8: return new Float8Datum(((DoubleWritable)value).get()); + + case TEXT: return new TextDatum(value.toString()); + } + + return NullDatum.get(); + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index e05a9c24ef..adc9622ef7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -33,6 +33,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.function.UDFInvocationDesc; +import org.apache.tajo.util.TajoHiveTypeConverter; import org.reflections.Reflections; import org.reflections.util.ConfigurationBuilder; @@ -117,13 +118,13 @@ static void extractUDFclasses(Set> classes, List 0) { params = new TajoDataTypes.DataType[hiveUDFparams.length]; for (int i=0; i> classes, List> classes, ArrayList list) { } - - private static TajoDataTypes.DataType convertHiveTypeToTajoType(Class hiveType) { - if (hiveType == null) - return null; - - if (hiveType == IntWritable.class) { - return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4); - } - if (hiveType == LongWritable.class) { - return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT8); - } - if (hiveType == Text.class) { - return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT); - } - if (hiveType == FloatWritable.class) { - return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT4); - } - if (hiveType == DoubleWritable.class) { - return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT8); - } - - return null; - } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java index af92c7f543..595d4eb351 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java @@ -18,13 +18,15 @@ package org.apache.tajo.plan.function; -import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.*; import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TextDatum; import org.apache.tajo.function.UDFInvocationDesc; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.TajoHiveTypeConverter; import java.io.IOException; import java.lang.reflect.Constructor; @@ -77,10 +79,15 @@ private Method getEvaluateMethod(Class clazz) { @Override public Datum eval(Tuple tuple) { Datum resultDatum = NullDatum.get(); + Writable [] params = new Writable[tuple.size()]; + + for (int i=0; i Date: Fri, 11 Dec 2015 16:07:00 +0900 Subject: [PATCH 36/72] Organize imports --- .../apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java | 2 -- .../java/org/apache/tajo/plan/function/HiveFunctionInvoke.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index adc9622ef7..c9ec6a1750 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -25,8 +25,6 @@ import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; -import org.apache.hadoop.io.*; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.FunctionDescBuilder; import org.apache.tajo.catalog.proto.CatalogProtos; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java index 595d4eb351..a8f5bbb062 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java @@ -20,10 +20,8 @@ import org.apache.hadoop.io.*; import org.apache.tajo.catalog.FunctionDesc; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.datum.TextDatum; import org.apache.tajo.function.UDFInvocationDesc; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.TajoHiveTypeConverter; From 49b4e811e9d99dd377fa062d2da23cdb051ff1c6 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Fri, 11 Dec 2015 16:24:37 +0900 Subject: [PATCH 37/72] Unused import --- tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index 53a22f8a16..0363594e8c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -65,7 +65,6 @@ import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.util.ArrayList; -import java.util.HashMap; import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.tajo.conf.TajoConf.ConfVars; From bca7b41abd725c7bcda5891c414649871af288da Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 14 Dec 2015 11:20:19 +0900 Subject: [PATCH 38/72] One more test added --- tajo-common/pom.xml | 4 ++++ .../tajo/util/TajoHiveTypeConverter.java | 14 +++++++----- .../hiveudf/TestHiveFunctionLoader.java | 20 +++++++++++++++--- .../src/test/resources/hiveudf/myudf.jar | Bin 1626 -> 2316 bytes tajo-core/pom.xml | 1 - tajo-project/pom.xml | 5 +++++ 6 files changed, 35 insertions(+), 9 deletions(-) diff --git a/tajo-common/pom.xml b/tajo-common/pom.xml index 06703b333d..cf4eb36deb 100644 --- a/tajo-common/pom.xml +++ b/tajo-common/pom.xml @@ -225,6 +225,10 @@ org.iq80.snappy snappy + + org.reflections + reflections + junit junit diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java b/tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java index 0dc1e8bce0..fd820cbb2a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java @@ -22,6 +22,9 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.*; +import org.reflections.ReflectionUtils; + +import java.util.Set; public class TajoHiveTypeConverter { @@ -30,20 +33,21 @@ public static DataType convertHiveTypeToTajoType(Class hiveType) { return null; DataType.Builder builder = DataType.newBuilder(); + Set> parents = ReflectionUtils.getAllSuperTypes(hiveType); - if (hiveType == IntWritable.class) { + if (hiveType == IntWritable.class || parents.contains(IntWritable.class)) { return builder.setType(Type.INT4).build(); } - if (hiveType == LongWritable.class) { + if (hiveType == LongWritable.class || parents.contains(LongWritable.class)) { return builder.setType(Type.INT8).build(); } - if (hiveType == Text.class) { + if (hiveType == Text.class || parents.contains(Text.class)) { return builder.setType(Type.TEXT).build(); } - if (hiveType == FloatWritable.class) { + if (hiveType == FloatWritable.class || parents.contains(FloatWritable.class)) { return builder.setType(Type.FLOAT4).build(); } - if (hiveType == DoubleWritable.class) { + if (hiveType == DoubleWritable.class || parents.contains(DoubleWritable.class)) { return builder.setType(Type.FLOAT8).build(); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 128e24ecf3..fd0a6cdf22 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -87,16 +87,23 @@ public void testFindFunction() throws Exception { FunctionDesc desc = catService.getFunction("my_upper", CatalogProtos.FunctionType.UDF, CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); - assertEquals(HiveGeneralFunctionHolder.class, desc.getLegacyFuncClass()); assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); assertEquals("to uppercase", desc.getDescription()); + TajoDataTypes.DataType int4type = CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4); + desc = catService.getFunction("my_divide", CatalogProtos.FunctionType.UDF, int4type, int4type); + + assertEquals(TajoDataTypes.Type.FLOAT8, desc.getReturnType().getType()); + assertEquals(2, desc.getParamTypes().length); + assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[0].getType()); + assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[1].getType()); + + // synonym desc = catService.getFunction("test_upper", CatalogProtos.FunctionType.UDF, CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); - assertEquals(HiveGeneralFunctionHolder.class, desc.getLegacyFuncClass()); assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); @@ -106,7 +113,6 @@ public void testFindFunction() throws Exception { desc = catService.getFunction("com_example_hive_udf_MyLower", CatalogProtos.FunctionType.UDF, CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); - assertEquals(HiveGeneralFunctionHolder.class, desc.getLegacyFuncClass()); assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); @@ -124,5 +130,13 @@ public void testRunFunctions() throws Exception { assertEquals("ABCD", result); rs.close(); + + rs = client.executeQueryAndGetResult("select my_divide(1,2)"); + rs.beforeFirst(); + rs.next(); + double res = rs.getDouble(1); + assertEquals(0.5, res, 0.0000001); + + rs.close(); } } \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/hiveudf/myudf.jar b/tajo-core-tests/src/test/resources/hiveudf/myudf.jar index b462126ac41e89476e74fb3c9f09eebeff58b072..45fe6cf8617005ce83f83bd912835a45e0a30ad3 100644 GIT binary patch delta 733 zcmcb`(<8(i;LXg!#lXP9z>pr%H<4GF8AKVYgQ-{@Fx6uq2o!W@Kmy8>&$CL^yJVJS zrljg6=Oh*v_Xgk2yX+ufJNt&G^rSCt4GasHwNBb17$*2==?WpGRU(>S8mB4DyKwBD z8uv!?8?RUVZ2HHyCxBDMuA$%Qj8*15ttQp;YtqlpxoK(r@9*>D`V7BsXe+QUZ)m@9 zDtUKvwh2%!smQWFcEp@(APTqNBX+_4y>G9ht!|yiW6Z@!rLY_MLXW zBP}rFw^wh>$~$E%?z;y`KWaYOaqR1aX?tvxzriK~u{=r110o#zkS zO8eNeu)bi?q!YsKm$t<0diCn1y4^FLtJ^(Sq$=z(ns(%q7SGgYQL84DWX1)Xgg!q# z;fm0_+d><~PflzMuGD_Q7#hK+CjWN}i|qXqZguHW1()(*FN#e_E&4&rng;N_R`d!?aZ9{Z;WbpK9Q`f{x0+;dRMH%Z0}~(I||Wrm`+ST zVLWA{k#`?w`sDTRYTci0@O;+#Rrn_V*6yy-g?fp#bA_k9EN4``6`^|R>y#fqO)A}T z!d6FkcZYIrben0Gm(;9xPv_z36&Fq|da-uOSF?8$)(h+V5-Mp@^cpL$+J1QHwQDHVFrm! zR$?~>(-G|EV0sd}4Vb>j?l3utLvFG@hcr-wDN_y8WJ3 org.reflections reflections - 0.9.10 diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index f8a82d056f..051da43822 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -1188,6 +1188,11 @@ jetty-util ${jetty.version} + + org.reflections + reflections + 0.9.10 + org.mortbay.jetty jsp-2.1 From 4e897100d8be8c321d6ba0800b3357c63b48c958 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 14 Dec 2015 12:22:31 +0900 Subject: [PATCH 39/72] Remove HiveGeneralFunctionHolder --- .../tajo/engine/function/FunctionLoader.java | 4 +- .../hiveudf/HiveGeneralFunctionHolder.java | 38 ------------------- 2 files changed, 1 insertion(+), 41 deletions(-) delete mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveGeneralFunctionHolder.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java index 8d33ea45bc..58ae070c94 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java @@ -36,7 +36,6 @@ import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamOptionTypes; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.engine.function.hiveudf.HiveGeneralFunctionHolder; import org.apache.tajo.function.*; import org.apache.tajo.plan.function.python.PythonScriptEngine; import org.apache.tajo.util.ClassUtil; @@ -87,7 +86,6 @@ public static Map load() { * Load functions defined by users. * * @param conf - * @param functionList * @return * @throws IOException */ @@ -219,7 +217,7 @@ public static List findLegacyFunctions() { Set functionClasses = ClassUtil.findClasses(Function.class, "org.apache.tajo.engine.function"); for (Class eachClass : functionClasses) { - if(eachClass.isInterface() || Modifier.isAbstract(eachClass.getModifiers()) || eachClass == HiveGeneralFunctionHolder.class) { + if(eachClass.isInterface() || Modifier.isAbstract(eachClass.getModifiers())) { continue; } Function function = null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveGeneralFunctionHolder.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveGeneralFunctionHolder.java deleted file mode 100644 index a922b0d4d9..0000000000 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveGeneralFunctionHolder.java +++ /dev/null @@ -1,38 +0,0 @@ -/*** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.function.hiveudf; - -import org.apache.tajo.catalog.Column; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.plan.function.GeneralFunction; -import org.apache.tajo.storage.Tuple; - -/** - * Class to wrap Hive Scalar UDF - */ -public class HiveGeneralFunctionHolder extends GeneralFunction { - public HiveGeneralFunctionHolder() { - super(new Column[0]); - } - - @Override - public Datum eval(Tuple params) { - return null; - } -} From e8c35385d0adb2b73964dc7c01bedb1d9c7f0729 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 14 Dec 2015 12:23:18 +0900 Subject: [PATCH 40/72] Confer added --- tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java | 3 +++ .../tajo/engine/function/hiveudf/TestHiveFunctionLoader.java | 2 +- .../tajo/engine/function/hiveudf/HiveFunctionLoader.java | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 9f788ebb8a..56d094e821 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -296,6 +296,9 @@ public static enum ConfVars implements ConfigKey { PYTHON_CODE_DIR("tajo.function.python.code-dir", ""), PYTHON_CONTROLLER_LOG_DIR("tajo.function.python.controller.log-dir", ""), + // HIVE UDF + HIVE_UDF_DIR("hive.udf.dir", "lib/hiveudf"), + // Partition PARTITION_DYNAMIC_BULK_INSERT_BATCH_SIZE("tajo.partition.dynamic.bulk-insert.batch-size", 1000), diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index fd0a6cdf22..2c493260f6 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -51,7 +51,7 @@ public final void setUp() throws Exception { URL hiveUDFURL = ClassLoader.getSystemResource("hiveudf"); Preconditions.checkNotNull(hiveUDFURL, "hive udf directory is absent."); - conf.set("hive.udf.dir", hiveUDFURL.toString().substring("file:".length())); + conf.set(TajoConf.ConfVars.HIVE_UDF_DIR.varname, hiveUDFURL.toString().substring("file:".length())); cluster.startMiniClusterInLocal(1); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index c9ec6a1750..e1bd11a305 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -43,7 +43,7 @@ public class HiveFunctionLoader { public static Collection loadHiveUDFs(TajoConf conf) { - String udfdir = conf.get("hive.udf.dir", "lib/hiveudf"); + String udfdir = conf.get(TajoConf.ConfVars.HIVE_UDF_DIR.varname); ArrayList funcList = new ArrayList<>(); try { From d6f4ee04c0fafc8e130960febc8b3c4444fa731e Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 14 Dec 2015 12:23:52 +0900 Subject: [PATCH 41/72] Comments and exception handling --- .../hiveudf/TestHiveFunctionLoader.java | 4 +--- .../function/hiveudf/HiveFunctionLoader.java | 21 ++++++++++++------- .../plan/function/HiveFunctionInvoke.java | 8 +++---- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 2c493260f6..5dcc206cb2 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -28,7 +28,6 @@ import org.apache.tajo.client.TajoClient; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.worker.TajoWorker; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -38,7 +37,6 @@ import java.net.URL; import java.sql.ResultSet; import java.util.*; -import java.util.stream.Stream; public class TestHiveFunctionLoader { private TajoTestingCluster cluster; @@ -67,7 +65,7 @@ public void testAnalyzeUDFclass() { funcSet.add(HiveUDFtest.class); List funcList = new LinkedList<>(); - HiveFunctionLoader.extractUDFclasses(funcSet, funcList, null); + HiveFunctionLoader.buildFunctionsFromUDF(funcSet, funcList, null); assertEquals(funcList.size(), 1); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index e1bd11a305..c8cb332d2e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -30,6 +30,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.function.UDFInvocationDesc; import org.apache.tajo.util.TajoHiveTypeConverter; import org.reflections.Reflections; @@ -54,25 +55,26 @@ public static Collection loadHiveUDFs(TajoConf conf) { return null; } + // loop each jar file for (FileStatus fstatus : localFS.listStatus(udfPath, (Path path) -> path.getName().endsWith(".jar"))) { URL[] urls = new URL[]{new URL("jar:" + fstatus.getPath().toUri().toURL() + "!/")}; - // For UDF's decendants (legacy) + // extract and register UDF's decendants (legacy Hive UDF form) Set> udfClasses = getSubclassesFromJarEntry(urls, UDF.class); if (udfClasses != null) { - extractUDFclasses(udfClasses, funcList, "jar:"+urls[0].getPath()); + buildFunctionsFromUDF(udfClasses, funcList, "jar:"+urls[0].getPath()); } - // For GenericUDF's decendants (newer interface) + // extract and register GenericUDF's decendants (newer interface for Hive UDF) Set> genericUDFclasses = getSubclassesFromJarEntry(urls, GenericUDF.class); if (genericUDFclasses != null) { - extractGenericUDFclasses(genericUDFclasses, funcList); + buildFunctionsFromGenericUDF(genericUDFclasses, funcList); } } } catch (IOException e) { - e.printStackTrace(); + throw new TajoInternalError(e); } return funcList; @@ -86,12 +88,14 @@ private static Set> getSubclassesFromJarEntry(URL[] urls, return refl.getSubTypesOf(targetCls); } - static void extractUDFclasses(Set> classes, List list, String jarurl) { + static void buildFunctionsFromUDF(Set> classes, List list, String jarurl) { for (Class clazz: classes) { String [] names; String value = null, extended = null; Description desc = clazz.getAnnotation(Description.class); + + // Check @Description annotation (if exists) if (desc != null) { names = desc.name().split(","); for (int i=0; i> classes, List> classes, List 0) { params = new TajoDataTypes.DataType[hiveUDFparams.length]; for (int i=0; i> classes, List> classes, List> classes, ArrayList list) { + private static void buildFunctionsFromGenericUDF(Set> classes, ArrayList list) { } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java index a8f5bbb062..a35e47283d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java @@ -21,7 +21,7 @@ import org.apache.hadoop.io.*; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.function.UDFInvocationDesc; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.TajoHiveTypeConverter; @@ -62,7 +62,7 @@ private Method getEvaluateMethod(Class clazz) { try { instance = constructor.newInstance(); } catch (InstantiationException|IllegalAccessException|InvocationTargetException e) { - e.printStackTrace(); + throw new TajoInternalError(e); } for (Method m: clazz.getMethods()) { @@ -76,7 +76,7 @@ private Method getEvaluateMethod(Class clazz) { @Override public Datum eval(Tuple tuple) { - Datum resultDatum = NullDatum.get(); + Datum resultDatum; Writable [] params = new Writable[tuple.size()]; for (int i=0; i Date: Tue, 15 Dec 2015 14:27:49 +0900 Subject: [PATCH 42/72] Some types are added --- ...verter.java => WritableTypeConverter.java} | 31 ++++++++++++------- .../function/hiveudf/HiveFunctionLoader.java | 6 ++-- .../plan/function/HiveFunctionInvoke.java | 6 ++-- 3 files changed, 26 insertions(+), 17 deletions(-) rename tajo-common/src/main/java/org/apache/tajo/util/{TajoHiveTypeConverter.java => WritableTypeConverter.java} (65%) diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java b/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java similarity index 65% rename from tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java rename to tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java index fd820cbb2a..24efec3b10 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/TajoHiveTypeConverter.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java @@ -26,28 +26,34 @@ import java.util.Set; -public class TajoHiveTypeConverter { +public class WritableTypeConverter { - public static DataType convertHiveTypeToTajoType(Class hiveType) { - if (hiveType == null) + public static DataType convertWritableToTajoType(Class writableClass) { + if (writableClass == null) return null; DataType.Builder builder = DataType.newBuilder(); - Set> parents = ReflectionUtils.getAllSuperTypes(hiveType); + Set> parents = ReflectionUtils.getAllSuperTypes(writableClass); - if (hiveType == IntWritable.class || parents.contains(IntWritable.class)) { + if (writableClass == ByteWritable.class || parents.contains(ByteWritable.class)) { + return builder.setType(Type.INT1).build(); + } + if (writableClass == ShortWritable.class || parents.contains(ShortWritable.class)) { + return builder.setType(Type.INT2).build(); + } + if (writableClass == IntWritable.class || parents.contains(IntWritable.class)) { return builder.setType(Type.INT4).build(); } - if (hiveType == LongWritable.class || parents.contains(LongWritable.class)) { + if (writableClass == LongWritable.class || parents.contains(LongWritable.class)) { return builder.setType(Type.INT8).build(); } - if (hiveType == Text.class || parents.contains(Text.class)) { + if (writableClass == Text.class || parents.contains(Text.class)) { return builder.setType(Type.TEXT).build(); } - if (hiveType == FloatWritable.class || parents.contains(FloatWritable.class)) { + if (writableClass == FloatWritable.class || parents.contains(FloatWritable.class)) { return builder.setType(Type.FLOAT4).build(); } - if (hiveType == DoubleWritable.class || parents.contains(DoubleWritable.class)) { + if (writableClass == DoubleWritable.class || parents.contains(DoubleWritable.class)) { return builder.setType(Type.FLOAT8).build(); } @@ -56,7 +62,8 @@ public static DataType convertHiveTypeToTajoType(Class hiveType) { public static Writable convertDatum2Writable(Datum value) { switch(value.type()) { - case INT2: return new IntWritable(value.asInt2()); + case INT1: return new ByteWritable(value.asByte()); + case INT2: return new ShortWritable(value.asInt2()); case INT4: return new IntWritable(value.asInt4()); case INT8: return new LongWritable(value.asInt8()); @@ -70,9 +77,11 @@ public static Writable convertDatum2Writable(Datum value) { } public static Datum convertWritable2Datum(Writable value) { - DataType type = convertHiveTypeToTajoType(value.getClass()); + DataType type = convertWritableToTajoType(value.getClass()); switch(type.getType()) { + case INT1: return new Int2Datum(((ByteWritable)value).get()); + case INT2: return new Int2Datum(((ShortWritable)value).get()); case INT4: return new Int4Datum(((IntWritable)value).get()); case INT8: return new Int8Datum(((LongWritable)value).get()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index c8cb332d2e..aa0a3c28bd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -32,7 +32,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.function.UDFInvocationDesc; -import org.apache.tajo.util.TajoHiveTypeConverter; +import org.apache.tajo.util.WritableTypeConverter; import org.reflections.Reflections; import org.reflections.util.ConfigurationBuilder; @@ -121,14 +121,14 @@ static void buildFunctionsFromUDF(Set> classes, List 0) { params = new TajoDataTypes.DataType[hiveUDFparams.length]; for (int i=0; i Date: Tue, 15 Dec 2015 16:24:23 +0900 Subject: [PATCH 43/72] remove Hive GenericUDF code --- .../engine/function/hiveudf/HiveFunctionLoader.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index aa0a3c28bd..fb6abe3ef2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.udf.UDFType; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.FunctionDescBuilder; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -65,13 +64,6 @@ public static Collection loadHiveUDFs(TajoConf conf) { if (udfClasses != null) { buildFunctionsFromUDF(udfClasses, funcList, "jar:"+urls[0].getPath()); } - - // extract and register GenericUDF's decendants (newer interface for Hive UDF) - Set> genericUDFclasses = getSubclassesFromJarEntry(urls, GenericUDF.class); - if (genericUDFclasses != null) { - buildFunctionsFromGenericUDF(genericUDFclasses, funcList); - } - } } catch (IOException e) { throw new TajoInternalError(e); @@ -159,7 +151,4 @@ static void buildFunctionsFromUDF(Set> classes, List> classes, ArrayList list) { - } } From aba597451d98ceab3fa7f8e76c5391f5167720e4 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 15 Dec 2015 17:42:09 +0900 Subject: [PATCH 44/72] config code modified --- .../src/main/java/org/apache/tajo/conf/TajoConf.java | 2 +- .../tajo/engine/function/hiveudf/HiveFunctionLoader.java | 2 +- .../src/main/java/org/apache/tajo/master/TajoMaster.java | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 56d094e821..a799c0f166 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -297,7 +297,7 @@ public static enum ConfVars implements ConfigKey { PYTHON_CONTROLLER_LOG_DIR("tajo.function.python.controller.log-dir", ""), // HIVE UDF - HIVE_UDF_DIR("hive.udf.dir", "lib/hiveudf"), + HIVE_UDF_DIR("hive.udf.dir", "./lib/hiveudf"), // Partition PARTITION_DYNAMIC_BULK_INSERT_BATCH_SIZE("tajo.partition.dynamic.bulk-insert.batch-size", 1000), diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index fb6abe3ef2..3c050738b3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -43,8 +43,8 @@ public class HiveFunctionLoader { public static Collection loadHiveUDFs(TajoConf conf) { - String udfdir = conf.get(TajoConf.ConfVars.HIVE_UDF_DIR.varname); ArrayList funcList = new ArrayList<>(); + String udfdir = conf.getVar(TajoConf.ConfVars.HIVE_UDF_DIR); try { FileSystem localFS = FileSystem.getLocal(conf); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index cf864c6c1b..521c5de253 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -224,8 +224,13 @@ private Collection loadFunctions() throws IOException, AmbiguousFu funcSet.put(desc.hashCodeWithoutType(), desc); } - checkUDFduplicates(udfs, funcSet); - checkUDFduplicates(hiveUDFs, funcSet); + if (udfs != null) { + checkUDFduplicates(udfs, funcSet); + } + + if (hiveUDFs != null) { + checkUDFduplicates(hiveUDFs, funcSet); + } return funcSet.values(); } From 7b78d2eb9b5914b1ec3d5c736307ed11f8674938 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 4 Jan 2016 14:27:23 +0900 Subject: [PATCH 45/72] Fixed the problem to miss to register some functions --- .../src/main/java/org/apache/tajo/master/TajoMaster.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index 33d6edebb1..4e2dc56d0f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -230,17 +230,17 @@ private Collection loadFunctions() throws IOException, AmbiguousFu } if (udfs != null) { - checkUDFduplicates(udfs, funcSet); + checkUDFduplicates(udfs, funcSet, functionList); } if (hiveUDFs != null) { - checkUDFduplicates(hiveUDFs, funcSet); + checkUDFduplicates(hiveUDFs, funcSet, functionList); } - return funcSet.values(); + return functionList; } - private void checkUDFduplicates(Collection udfs, HashMap funcSet) + private void checkUDFduplicates(Collection udfs, HashMap funcSet, List funcList) throws AmbiguousFunctionException { for (FunctionDesc desc: udfs) { if (funcSet.containsKey(desc.hashCodeWithoutType())) { @@ -248,6 +248,7 @@ private void checkUDFduplicates(Collection udfs, HashMap Date: Tue, 5 Jan 2016 17:10:33 +0900 Subject: [PATCH 46/72] code changed slightly --- .../java/org/apache/tajo/engine/function/FunctionLoader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java index c356d87fa0..afb3411911 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java @@ -117,8 +117,8 @@ public static List loadUserDefinedFunctions(TajoConf conf) filePaths.add(codePath); } for (Path filePath : filePaths) { - PythonScriptEngine.registerFunctions(filePath.toUri(), FunctionLoader.PYTHON_FUNCTION_NAMESPACE) - .stream().map(functionList::add); + PythonScriptEngine.registerFunctions(filePath.toUri(), FunctionLoader.PYTHON_FUNCTION_NAMESPACE). + forEach(functionList::add); } } } From cf2bcd0a658ba753de29f036a055f22e7b3a2822 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 5 Jan 2016 17:46:02 +0900 Subject: [PATCH 47/72] TAJO-2034: Files required for executing python functions are not copied in testEval(). --- .../org/apache/tajo/engine/eval/ExprTestBase.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 600a45f38d..41ff85cfd8 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -43,6 +43,8 @@ import org.apache.tajo.plan.*; import org.apache.tajo.plan.expr.EvalContext; import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.function.PythonFunctionInvoke; +import org.apache.tajo.plan.function.python.PythonScriptEngine; import org.apache.tajo.plan.serder.EvalNodeDeserializer; import org.apache.tajo.plan.serder.EvalNodeSerializer; import org.apache.tajo.plan.serder.PlanProto; @@ -60,6 +62,7 @@ import org.junit.AfterClass; import org.junit.BeforeClass; +import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; @@ -276,6 +279,9 @@ public void testEval(OverridableConf context, Schema schema, String tableName, S EvalContext evalContext = new EvalContext(); try { + if (needPythonFileCopy()) { + PythonScriptEngine.initPythonScriptEngineFiles(); + } targets = getRawTargets(queryContext, query, condition); EvalCodeGenerator codegen = null; @@ -333,6 +339,11 @@ public void testEval(OverridableConf context, Schema schema, String tableName, S } } + private static boolean needPythonFileCopy() { + File contoller = new File(PythonScriptEngine.getControllerPath()); + return !contoller.exists(); + } + public static void assertEvalTreeProtoSerDer(OverridableConf context, EvalNode evalNode) { PlanProto.EvalNodeTree converted = EvalNodeSerializer.serialize(evalNode); assertEquals(evalNode, EvalNodeDeserializer.deserialize(context, null, converted)); From e224c336db4b536f2b91099b66520e6e54bebf80 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 5 Jan 2016 17:47:33 +0900 Subject: [PATCH 48/72] Optimize import --- .../java/org/apache/tajo/engine/eval/ExprTestBase.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 41ff85cfd8..c50a25f6fa 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -34,16 +34,18 @@ import org.apache.tajo.engine.codegen.TajoClassLoader; import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.function.FunctionSignature; import org.apache.tajo.master.exec.QueryExecutor; -import org.apache.tajo.plan.*; +import org.apache.tajo.parser.sql.SQLAnalyzer; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.EvalContext; import org.apache.tajo.plan.expr.EvalNode; -import org.apache.tajo.plan.function.PythonFunctionInvoke; import org.apache.tajo.plan.function.python.PythonScriptEngine; import org.apache.tajo.plan.serder.EvalNodeDeserializer; import org.apache.tajo.plan.serder.EvalNodeSerializer; From d3304a92fd31e751563049abde13a74d78ca2010 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 7 Jan 2016 10:59:43 +0900 Subject: [PATCH 49/72] comment --- .../main/java/org/apache/tajo/function/FunctionSignature.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java index b5b0cbdf8b..7276d45f77 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java @@ -41,6 +41,8 @@ public class FunctionSignature implements Comparable, ProtoOb private DataType[] paramTypes; @Expose private DataType returnType; + + // for future use @Expose private boolean deterministic = true; From 02f5b3280086b95d9eae4d406a3ed220cb6691ca Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Fri, 15 Jan 2016 16:15:56 +0900 Subject: [PATCH 50/72] BytesWritable added --- .../org/apache/tajo/util/WritableTypeConverter.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java b/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java index 24efec3b10..411c7e392f 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java @@ -22,6 +22,8 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.*; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoRuntimeException; import org.reflections.ReflectionUtils; import java.util.Set; @@ -56,8 +58,11 @@ public static DataType convertWritableToTajoType(Class writa if (writableClass == DoubleWritable.class || parents.contains(DoubleWritable.class)) { return builder.setType(Type.FLOAT8).build(); } + if (writableClass == BytesWritable.class || parents.contains(BytesWritable.class)) { + return builder.setType(Type.VARBINARY).build(); + } - return builder.setType(Type.NULL_TYPE).build(); + throw new TajoRuntimeException(new NotImplementedException(writableClass.getSimpleName())); } public static Writable convertDatum2Writable(Datum value) { @@ -71,9 +76,10 @@ public static Writable convertDatum2Writable(Datum value) { case FLOAT8: return new DoubleWritable(value.asFloat8()); case TEXT: return new Text(value.asChars()); + case VARBINARY: return new BytesWritable(value.asByteArray()); } - return NullWritable.get(); + throw new TajoRuntimeException(new NotImplementedException(value.type().name())); } public static Datum convertWritable2Datum(Writable value) { @@ -89,6 +95,7 @@ public static Datum convertWritable2Datum(Writable value) { case FLOAT8: return new Float8Datum(((DoubleWritable)value).get()); case TEXT: return new TextDatum(value.toString()); + case VARBINARY: return new BlobDatum(((BytesWritable)value).getBytes()); } return NullDatum.get(); From 43e2201d9a1a83a939033927706db96839ad0a40 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Fri, 12 Feb 2016 16:00:09 +0900 Subject: [PATCH 51/72] move params variable to instance field --- .../java/org/apache/tajo/plan/function/HiveFunctionInvoke.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java index 892ff649ce..36c6e8a823 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java @@ -36,9 +36,11 @@ public class HiveFunctionInvoke extends FunctionInvoke implements Cloneable { private Object instance = null; private Method evalMethod = null; + private Writable [] params; public HiveFunctionInvoke(FunctionDesc desc) { super(desc); + params = new Writable[desc.getParamTypes().length]; } @Override @@ -77,7 +79,6 @@ private Method getEvaluateMethod(Class clazz) { @Override public Datum eval(Tuple tuple) { Datum resultDatum; - Writable [] params = new Writable[tuple.size()]; for (int i=0; i Date: Fri, 12 Feb 2016 16:09:09 +0900 Subject: [PATCH 52/72] Consider other filesystems --- .../tajo/engine/function/hiveudf/HiveFunctionLoader.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index 3c050738b3..8e66a18c75 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -47,15 +47,15 @@ public static Collection loadHiveUDFs(TajoConf conf) { String udfdir = conf.getVar(TajoConf.ConfVars.HIVE_UDF_DIR); try { - FileSystem localFS = FileSystem.getLocal(conf); Path udfPath = new Path(udfdir); + FileSystem fs = udfPath.getFileSystem(conf); - if (!localFS.isDirectory(udfPath)) { + if (!fs.isDirectory(udfPath)) { return null; } // loop each jar file - for (FileStatus fstatus : localFS.listStatus(udfPath, (Path path) -> path.getName().endsWith(".jar"))) { + for (FileStatus fstatus : fs.listStatus(udfPath, (Path path) -> path.getName().endsWith(".jar"))) { URL[] urls = new URL[]{new URL("jar:" + fstatus.getPath().toUri().toURL() + "!/")}; From 082d5d2ff71a59c34880af7e3e85d55426fd8522 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Wed, 17 Feb 2016 15:22:38 +0900 Subject: [PATCH 53/72] Date and Timestamp types added for Hive UDF --- tajo-common/pom.xml | 6 ++ .../tajo/util/WritableTypeConverter.java | 24 +++++++ .../tajo/util/datetime/DateTimeConstants.java | 18 ++--- .../tajo/util/datetime/DateTimeUtil.java | 46 ++++--------- .../tajo/util/TestWritableTypeConverter.java | 65 +++++++++++++++++++ .../apache/tajo/storage/orc/ORCScanner.java | 3 +- .../storage/thirdparty/orc/WriterImpl.java | 3 +- 7 files changed, 120 insertions(+), 45 deletions(-) create mode 100644 tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java diff --git a/tajo-common/pom.xml b/tajo-common/pom.xml index cf4eb36deb..b999a8b5c4 100644 --- a/tajo-common/pom.xml +++ b/tajo-common/pom.xml @@ -249,6 +249,12 @@ powermock-api-mockito test + + org.apache.hive + hive-exec + ${hive.version} + provided + diff --git a/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java b/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java index 411c7e392f..dda115a1ef 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java @@ -18,14 +18,19 @@ package org.apache.tajo.util; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.*; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.*; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.util.datetime.DateTimeConstants; +import org.apache.tajo.util.datetime.DateTimeUtil; import org.reflections.ReflectionUtils; +import java.sql.Timestamp; import java.util.Set; public class WritableTypeConverter { @@ -58,6 +63,12 @@ public static DataType convertWritableToTajoType(Class writa if (writableClass == DoubleWritable.class || parents.contains(DoubleWritable.class)) { return builder.setType(Type.FLOAT8).build(); } + if (writableClass == DateWritable.class || parents.contains(DateWritable.class)) { + return builder.setType(Type.DATE).build(); + } + if (writableClass == TimestampWritable.class || parents.contains(TimestampWritable.class)) { + return builder.setType(Type.TIMESTAMP).build(); + } if (writableClass == BytesWritable.class || parents.contains(BytesWritable.class)) { return builder.setType(Type.VARBINARY).build(); } @@ -75,6 +86,15 @@ public static Writable convertDatum2Writable(Datum value) { case FLOAT4: return new FloatWritable(value.asFloat4()); case FLOAT8: return new DoubleWritable(value.asFloat8()); + // NOTE: value should be DateDatum + case DATE: return new DateWritable(value.asInt4() - DateTimeConstants.UNIX_EPOCH_JDATE); + + // NOTE: value should be TimestampDatum + case TIMESTAMP: + TimestampWritable result = new TimestampWritable(); + result.setTime(DateTimeUtil.julianTimeToJavaTime(value.asInt8())); + return result; + case TEXT: return new Text(value.asChars()); case VARBINARY: return new BytesWritable(value.asByteArray()); } @@ -94,6 +114,10 @@ public static Datum convertWritable2Datum(Writable value) { case FLOAT4: return new Float4Datum(((FloatWritable)value).get()); case FLOAT8: return new Float8Datum(((DoubleWritable)value).get()); + case DATE: return new DateDatum(((DateWritable)value).getDays() + DateTimeConstants.UNIX_EPOCH_JDATE); + case TIMESTAMP: return new TimestampDatum(DateTimeUtil.javaTimeToJulianTime( + ((TimestampWritable)value).getTimestamp().getTime())); + case TEXT: return new TextDatum(value.toString()); case VARBINARY: return new BlobDatum(((BytesWritable)value).getBytes()); } diff --git a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeConstants.java b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeConstants.java index e610deeacd..002ae0bbc7 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeConstants.java @@ -55,18 +55,18 @@ public enum DateStyle { /** avoid floating-point computation */ public static final int SECS_PER_YEAR = 36525 * 864; - public static final int SECS_PER_DAY = 86400; + public static final int SECS_PER_DAY = 86_400; public static final int SECS_PER_HOUR = 3600; public static final int SECS_PER_MINUTE = 60; public static final int MINS_PER_HOUR = 60; - public static final long MSECS_PER_DAY = 86400000L; + public static final long MSECS_PER_DAY = 86_400_000L; public static final long MSECS_PER_SEC = 1000L; - public static final long USECS_PER_DAY = 86400000000L; - public static final long USECS_PER_HOUR = 3600000000L; - public static final long USECS_PER_MINUTE = 60000000L; - public static final long USECS_PER_SEC = 1000000L; + public static final long USECS_PER_DAY = 86_400_000_000L; + public static final long USECS_PER_HOUR = 3_600_000_000L; + public static final long USECS_PER_MINUTE = 60_000_000L; + public static final long USECS_PER_SEC = 1_000_000L; public static final long USECS_PER_MSEC = 1000L; public static final int JULIAN_MINYEAR = -4713; @@ -100,11 +100,11 @@ public enum DateStyle { // Julian-date equivalents of Day 0 in Unix and Postgres reckoning /** == DateTimeUtil.toJulianDate(1970, 1, 1) */ - public static final int UNIX_EPOCH_JDATE = 2440588; + public static final int UNIX_EPOCH_JDATE = 2_440_588; /** == DateTimeUtil.toJulianDate(2000, 1, 1) */ - public static final int POSTGRES_EPOCH_JDATE = 2451545; + public static final int POSTGRES_EPOCH_JDATE = 2_451_545; /** == (POSTGRES_EPOCH_JDATE * SECS_PER_DAY) - (UNIX_EPOCH_JDATE * SECS_PER_DAY); */ - public static final long SECS_DIFFERENCE_BETWEEN_JULIAN_AND_UNIXTIME = 946684800; + public static final long SECS_DIFFERENCE_BETWEEN_JULIAN_AND_UNIXTIME = 946_684_800; public static final int MAX_TZDISP_HOUR = 15; /* maximum allowed hour part */ public static final int TZDISP_LIMIT = ((MAX_TZDISP_HOUR + 1) * SECS_PER_HOUR); diff --git a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java index 5a338d39a5..79ef391b7b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeUtil.java @@ -40,8 +40,6 @@ public class DateTimeUtil { /** maximum possible number of fields in a date * string */ private static int MAXDATEFIELDS = 25; - public final static int DAYS_FROM_JULIAN_TO_EPOCH = 2440588; - public static boolean isJulianCalendar(int year, int month, int day) { return year <= 1752 && month <= 9 && day < 14; } @@ -304,12 +302,12 @@ public static int julianTimeToEpoch(long timestamp) { /** * Converts julian timestamp to java timestamp. - * @param timestamp - * @return + * @param timestamp julian time in millisecond + * @return java time in millisecond */ public static long julianTimeToJavaTime(long timestamp) { double totalSecs = (double)timestamp / (double)DateTimeConstants.MSECS_PER_SEC; - return (long)(Math.round(totalSecs + DateTimeConstants.SECS_DIFFERENCE_BETWEEN_JULIAN_AND_UNIXTIME * 1000.0)); + return Math.round(totalSecs + DateTimeConstants.SECS_DIFFERENCE_BETWEEN_JULIAN_AND_UNIXTIME * 1000.0); } /** @@ -2107,43 +2105,28 @@ public static String getTimeZoneDisplayTime(int totalSecs) { } public static long getDay(TimeMeta dateTime) { - long usecs = 0; - - usecs = julianTimeToJavaTime(toJulianTimestamp(dateTime.years, dateTime.monthOfYear, dateTime.dayOfMonth, + return julianTimeToJavaTime(toJulianTimestamp(dateTime.years, dateTime.monthOfYear, dateTime.dayOfMonth, 0, 0, 0, 0)) * DateTimeConstants.USECS_PER_MSEC; - return usecs; } public static long getHour(TimeMeta dateTime) { - long usecs = 0; - - usecs = julianTimeToJavaTime(toJulianTimestamp(dateTime.years, dateTime.monthOfYear, dateTime.dayOfMonth, + return julianTimeToJavaTime(toJulianTimestamp(dateTime.years, dateTime.monthOfYear, dateTime.dayOfMonth, dateTime.hours, 0, 0, 0)) * DateTimeConstants.USECS_PER_MSEC; - return usecs; } public static long getMinute(TimeMeta dateTime) { - long usecs = 0; - - usecs = julianTimeToJavaTime(toJulianTimestamp(dateTime.years, dateTime.monthOfYear, dateTime.dayOfMonth, + return julianTimeToJavaTime(toJulianTimestamp(dateTime.years, dateTime.monthOfYear, dateTime.dayOfMonth, dateTime.hours, dateTime.minutes, 0, 0)) * DateTimeConstants.USECS_PER_MSEC; - return usecs; } public static long getSecond(TimeMeta dateTime) { - long usecs = 0; - - usecs = julianTimeToJavaTime(toJulianTimestamp(dateTime.years, dateTime.monthOfYear, dateTime.dayOfMonth, + return julianTimeToJavaTime(toJulianTimestamp(dateTime.years, dateTime.monthOfYear, dateTime.dayOfMonth, dateTime.hours, dateTime.minutes, dateTime.secs, 0)) * DateTimeConstants.USECS_PER_MSEC; - return usecs; } public static long getMonth(TimeMeta dateTime) { - long usecs = 0; - - usecs = julianTimeToJavaTime(toJulianTimestamp(dateTime.years, dateTime.monthOfYear, 1, 0, 0, 0, 0)) * + return julianTimeToJavaTime(toJulianTimestamp(dateTime.years, dateTime.monthOfYear, 1, 0, 0, 0, 0)) * DateTimeConstants.USECS_PER_MSEC; - return usecs; } public static long getDayOfWeek(TimeMeta dateTime, int weekday) { @@ -2153,22 +2136,17 @@ public static long getDayOfWeek(TimeMeta dateTime, int weekday) { int week = date2isoweek(dateTime.years, dateTime.monthOfYear, dateTime.dayOfMonth); int jday = isoweek2j(dateTime.years, week); - long usecs = 0; - + jday += (weekday - 1); jday -= DateTimeConstants.POSTGRES_EPOCH_JDATE; - usecs = julianTimeToJavaTime(toJulianTimestamp(jday, 0, 0, 0, 0)) * - DateTimeConstants.USECS_PER_MSEC; - return usecs; + + return julianTimeToJavaTime(toJulianTimestamp(jday, 0, 0, 0, 0)) * DateTimeConstants.USECS_PER_MSEC; } public static long getYear(TimeMeta dateTime) { - long usecs = 0; - - usecs = julianTimeToJavaTime(toJulianTimestamp(dateTime.years, 1, 1, 0, 0, 0, 0)) * + return julianTimeToJavaTime(toJulianTimestamp(dateTime.years, 1, 1, 0, 0, 0, 0)) * DateTimeConstants.USECS_PER_MSEC; - return usecs; } public static TimeMeta getUTCDateTime(Int8Datum int8Datum){ diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java b/tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java new file mode 100644 index 0000000000..0d9378ab56 --- /dev/null +++ b/tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.util; + +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.io.Writable; +import org.apache.tajo.datum.DateDatum; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.TimestampDatum; +import org.apache.tajo.util.datetime.DateTimeUtil; +import org.apache.tajo.util.datetime.TimeMeta; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class TestWritableTypeConverter { + @Test + public void testDateTypeConverting() { + Datum testDatum; + TimeMeta testTM = new TimeMeta(); + + testTM.years = 1977; + testTM.monthOfYear = 6; + testTM.dayOfMonth = 28; + + testDatum = new DateDatum(testTM); + + Writable resultWritable = WritableTypeConverter.convertDatum2Writable(testDatum); + assertEquals("1977-06-28", ((DateWritable)resultWritable).get().toString()); + + Datum resultDatum = WritableTypeConverter.convertWritable2Datum(resultWritable); + assertEquals(testDatum, resultDatum); + } + + @Test + public void testTimestampTypeConverting() { + Datum testDatum; + long currentMills = System.currentTimeMillis(); + + testDatum = new TimestampDatum(DateTimeUtil.javaTimeToJulianTime(currentMills)); + + Writable resultWritable = WritableTypeConverter.convertDatum2Writable(testDatum); + assertEquals(currentMills / 1000, ((TimestampWritable)resultWritable).getSeconds()); + + Datum resultDatum = WritableTypeConverter.convertWritable2Datum(resultWritable); + assertEquals(testDatum, resultDatum); + } +} diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java index 9351c59926..3616bc3451 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java @@ -42,6 +42,7 @@ import com.facebook.presto.orc.*; import com.facebook.presto.orc.metadata.OrcMetadataReader; import org.apache.tajo.storage.thirdparty.orc.HdfsOrcDataSource; +import org.apache.tajo.util.datetime.DateTimeConstants; import org.apache.tajo.util.datetime.DateTimeUtil; import org.joda.time.DateTimeZone; @@ -269,7 +270,7 @@ private Datum createValueDatum(Vector vector, TajoDataTypes.DataType type) { return NullDatum.get(); return DatumFactory.createDate( - (int) ((LongVector) vector).vector[currentPosInBatch] + DateTimeUtil.DAYS_FROM_JULIAN_TO_EPOCH); + (int) ((LongVector) vector).vector[currentPosInBatch] + DateTimeConstants.UNIX_EPOCH_JDATE); case INET4: if (((LongVector) vector).isNull[currentPosInBatch]) diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java index 833d102744..39cc15d11b 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.io.Text; +import org.apache.tajo.util.datetime.DateTimeConstants; import org.apache.tajo.util.datetime.DateTimeUtil; import java.io.IOException; @@ -1576,7 +1577,7 @@ private static class DateTreeWriter extends TreeWriter { void write(Datum datum) throws IOException { super.write(datum); if (datum != null && datum.isNotNull()) { - int daysSinceEpoch = datum.asInt4() - DateTimeUtil.DAYS_FROM_JULIAN_TO_EPOCH; + int daysSinceEpoch = datum.asInt4() - DateTimeConstants.UNIX_EPOCH_JDATE; // Using the Writable here as it's used directly for writing as well as for stats. indexStatistics.updateDate(daysSinceEpoch); writer.write(daysSinceEpoch); From 7bab5fbca8cd7f7c66b826ad175135f9bfa192a4 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Wed, 24 Feb 2016 16:01:22 +0900 Subject: [PATCH 54/72] multiple 'evaluate's are handled --- .../hiveudf/TestHiveFunctionLoader.java | 11 ++- .../src/test/resources/hiveudf/my_udf_src.zip | Bin 0 -> 3305 bytes .../src/test/resources/hiveudf/myudf.jar | Bin 2316 -> 3129 bytes .../function/hiveudf/HiveFunctionLoader.java | 63 ++++++++++-------- .../plan/function/HiveFunctionInvoke.java | 22 +++++- 5 files changed, 64 insertions(+), 32 deletions(-) create mode 100644 tajo-core-tests/src/test/resources/hiveudf/my_udf_src.zip diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 5dcc206cb2..09bf53e896 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -107,13 +107,22 @@ public void testFindFunction() throws Exception { assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); assertEquals("to uppercase", desc.getDescription()); - // Test for UDF without @Description + // Test for UDF without @Description and including multi 'evaluate()' desc = catService.getFunction("com_example_hive_udf_MyLower", CatalogProtos.FunctionType.UDF, CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); assertEquals(1, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); + + // same function for another parameter signature + desc = catService.getFunction("com_example_hive_udf_MyLower", CatalogProtos.FunctionType.UDF, + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); + + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); + assertEquals(2, desc.getParamTypes().length); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[1].getType()); } @Test diff --git a/tajo-core-tests/src/test/resources/hiveudf/my_udf_src.zip b/tajo-core-tests/src/test/resources/hiveudf/my_udf_src.zip new file mode 100644 index 0000000000000000000000000000000000000000..1f61a865b0164e1e29da0b3925696945763b388f GIT binary patch literal 3305 zcmWIWW@h1H0D%v2+U{Tmlwe^HU??t1)(?%~VR-I(Hf*uer_isAA`AgIb#Ov;tX=+kf8suz|?&_dm7eY_}RHy^CVucfH~0-{pRC()D*6O*%89!c+e5j@-1- zH}Qru*ZugKdBR27RxFv-ebyJPduyD9KLu_zO}=9NnmOWr;lq|YF2NevvFB=D-Ecf9 zEwe;j{KTwZ^QUfq_R?*NPX3!6*M7Xy?D(;NZtTngzD>uu=S*_^)AK@5Zo!M^{I0^9 zI%1ylTjVBB49z&n&v=66&4t5NZH!AzUoDY|-oc$AvTF6wqOYHJHT(!+T|J}wjf!KI zAg|@XV~6f%-3RL@TCU#0#U8WZcgOTkdkzPL zeZ3j!rY`Om_Q|I0qO`$}uiYzpqRRtTsf2CuiLTz3ZXe*y&T)ZV!(D)pfx(7}0h}!$ zp^qM&`oxC5Pkwo75lZ00qPDmHtlwb+ftL4mUADKhHcg3JJg>{*qVNZX?#-UMGPi#$ z*UkF;u2e|tg<{j}^5?VXZgl_JDBD=HcZu+~dlT-wWsx|0qulyKV|N@F&YR#pf&byWHTi@i#?lbRrbKNS7netp@|51;a zj$IOL-{yaL#aP$>dbiotwR|ypuD>+`XhQY{g=On^yQ3P~(W`12HH3sjJ6Lrx zcDpqjd#<(G_R&AHwW?f3-D8bFT<`PR{Aa!HrH3Xu?DV|Es-Ev5_jP8(oR>Cnf!|GvI_dU3{#1iSsxYaJwe z1m~@9kBpZU^h)}nWgM1qj@@MaA(xs9VrMUPFG^ylVvt+-`aoJT3yYWKjbEDOXAk(d z+pPx0gC1+HyPj9%_50Ib#B^+v$a`~aA0sG~(|0f1`4bq*>rq0vCan)8lud|BvB4!p znR)5X`FY8SB`C@EwoM*5sHXsK0+A4!Bn`?|>9zTw4D z@b~=-sU30E4(l&Zp7-TTisi!CWaotB^o%2Vl}5V!DOKBNg^HGIrZvuxZsyEg?OI~E zY~}N-tJ~I8&bc4+Y}tE_6JIklIgcAyuY2opVc~+E!H+~1zDtwhnsRg2ZO+oxw3 zFYfF-Exzf{LY3ATXOm{%|NrFIi!1uJ(HrEv8`)3F%vvp>!*xMJB&Kn$W=$A_cz0t_ zn!8keH~%swb}_f}@*(n5=FL-lXZ)0B$=Q!<59GJswPH${WXI02_q(yG+12`L2HqFd zBCDQnxg=I3y#8KZ|BcA^e@|x4yL@ti;>==Cx%=vi<~i7&HVas9e=Kj6J}3&FJZW|m z1P1?fXrY0<_7z5sf}+&o{L-T2)M9X%4=R<3uZ&Ali%Y=TB#F`ntBk?g;FU3|m5fZH z%-CJc!oUNo$Q<4}f|!sN1P8nY5#WtmHz$g2BdBg@YXZ9w=H?D!D4@YX1@i8CCnRl)&pl_1*$@&61fJ3ScL8+P=$nCh@x6Fm4!%)kgF$%MWE^l*+-x<8M&x1LGjUV zR-!DzQbI#)1C`Lo$pF3aFN|c{TgN(fB-@Y+BZy(3!U&&XsQG$7)G%nl#K8b7nD7~a gnyM3?XoeGr(6i|a?UvIK3VN(uD^$)jK-^zPbX~)>c5@h*)zw&!cXJ>su-b8 zFB!*2?hDW5t?@9Kz-)KsOyB*=xbvs>*VX^#OOW3!HZ8#9;|(V!Rc`+c3s?CHM~m@@ zgjTF_I(fuc_&n#a?DfxG^EplyJ?)ub#S>LOX+rVdc{TQL;!b>UiTQEoxQdD+7B;VEH9yj%_LXPz! z{aLcty21p$$m$pPJxL8&uT$Z!uq`TnMTSn(|Lyrn8`d9UJeb9Dc!OrY&DK4#@4tUq zEWs&l_O4#&+P{+3hWzzvnmY4BHbeJ3GLBWz{5w(p;I%HANP}&%c~w`7*VerKnzp_pZr?GZ*J^74TP{ zYxy0~+1@WX?Lo|ne=f7ut($1EzHg$8vZ>>*EVj*CG!9#MGt2w5CoNdTbw{~3uJ@wy z5(&OHq4fvWoc(a!;GDWa@^nL^t&yh3=Gw<>uChD5&+^;Z?B~6j#n1M%x8I5k&HlBU zqu(U)FI#{&J4c(u#pjbH)wPISEJ~>Ki0P_W zDx&3Dpz-8TX0rd>wKD4?tpgt%vu_d>Vy|Pc=Q6AdkaAfOkW-od{PxeZ{WD|h|9|`{ z-C(^V?kLYXiyf~PXa-a+aGq-;GUMi*R~%EKk101+)kn8~DZ21}QtjGZjLN5TAFo=l zc#Yp*CLMKKS?)F4V=qrP3Hqy{aPzYBBrS_<^V-I`>ZMy0>n~wU;LweDYV)_kF-D_u^C^T_0Osd@bgUhXfc^p>+<^qWfb z!Lo19#5>O0XdZmHaZ+RPAEvD~44+qQ5{*b$GG*fX&EZ~_k1orfY3Ok50z;%c%oyAh-~iexu~FebBGLl~FX?I8?N4qrY{ znFdL;pfo$VfWt`vB;>nzwe~Yc1_oDPGFC!Lo(v43tdk8nB!LN;SBsHJgaMJhk)5NC eqNST1s6|`@Gf@Y4v$BCyasZ(P&?m1sK|BDQ_)^&b delta 598 zcmdlf(Idnc;LXg!#lXP9z>pr%=RT27i3LoXs6psBZ3x|~52iOBh-G5=nOERGS&`YO zzBcf*ACsfNnrB;gtkY+ydHk4VlEufgrjJb@w^Rjs+Vq{~NETze&iZxzmqndwalU^v zlHWVoWTf!jwKWaBdUxje_1DW8cHh{PxG-U1hH@jTbazD8s#I=mu`te=wRf*RWK@h? zXTq^^SEkmXp4GcwExErlG(V*NYTnQw$v=euckH;8o>*&1V+Sbz^qR)xvDSZU1H+T_Qc- z^_1}QcK1D18GiML9m=CyYIZ(y+sqMP7M;}b>B8pyhKEuz9?6wfp6(VoRi8SQ&1~Ys z^}T*fB5RLdd^Y=m&z#G*^%pGvQ8_7Djm@~g!pfOxnZena`=vK4e>9wQhMnsw|B{y< zc{{sb8BaRzu~*^mkL0xDOLT7E+?ZIXAtmW{Ri@~p%;$}2|9Jzv*`+zo+}xa4#mK-g zfr)_u6p> classes, List 0) { - params = new TajoDataTypes.DataType[hiveUDFparams.length]; - for (int i=0; i> classes, List> classes, List list) { + TajoDataTypes.DataType retType = + WritableTypeConverter.convertWritableToTajoType((Class)method.getReturnType()); + TajoDataTypes.DataType[] params = convertTajoParamterTypes(method.getParameterTypes()); + + builder.setReturnType(retType).setParams(params); + + for (String name: names) { + builder.setName(name); + builder.setUDF(udfInvocation); + list.add(builder.build()); + } + } + + private static TajoDataTypes.DataType[] convertTajoParamterTypes(Class[] hiveUDFparams) { + TajoDataTypes.DataType [] params = null; + + // convert types to ones of Tajo + if (hiveUDFparams != null && hiveUDFparams.length > 0) { + params = new TajoDataTypes.DataType[hiveUDFparams.length]; + for (int i=0; i udfclass = loader.loadClass(udfDesc.getName()); - evalMethod = getEvaluateMethod(udfclass); + evalMethod = getEvaluateMethod(functionDesc.getParamTypes(), udfclass); } catch (ClassNotFoundException e) { e.printStackTrace(); } } - private Method getEvaluateMethod(Class clazz) { + private Method getEvaluateMethod(TajoDataTypes.DataType [] paramTypes, Class clazz) { Constructor constructor = clazz.getConstructors()[0]; try { @@ -69,13 +70,28 @@ private Method getEvaluateMethod(Class clazz) { for (Method m: clazz.getMethods()) { if (m.getName().equals("evaluate")) { - return m; + Class [] methodParamTypes = m.getParameterTypes(); + if (checkParamTypes(methodParamTypes, paramTypes)) { + return m; + } } } return null; } + boolean checkParamTypes(Class [] writableParams, TajoDataTypes.DataType [] tajoParams) { + int i = 0; + + for (Class writable: writableParams) { + if (!WritableTypeConverter.convertWritableToTajoType(writable).equals(tajoParams[i++])) { + return false; + } + } + + return true; + } + @Override public Datum eval(Tuple tuple) { Datum resultDatum; From 9472e8abdf15cc39ab970b86b84a7d60bc7c3971 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 7 Mar 2016 14:18:42 +0900 Subject: [PATCH 55/72] Error processing refined --- .../main/java/org/apache/tajo/util/WritableTypeConverter.java | 3 ++- .../java/org/apache/tajo/plan/function/HiveFunctionInvoke.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java b/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java index dda115a1ef..80fd77f965 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java @@ -26,6 +26,7 @@ import org.apache.tajo.datum.*; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedDataTypeException; import org.apache.tajo.util.datetime.DateTimeConstants; import org.apache.tajo.util.datetime.DateTimeUtil; import org.reflections.ReflectionUtils; @@ -122,6 +123,6 @@ public static Datum convertWritable2Datum(Writable value) { case VARBINARY: return new BlobDatum(((BytesWritable)value).getBytes()); } - return NullDatum.get(); + throw new TajoRuntimeException(new UnsupportedDataTypeException(value.getClass().getTypeName())); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java index 1b24a9f2f1..89410999e8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java @@ -23,6 +23,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.function.UDFInvocationDesc; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.WritableTypeConverter; @@ -55,7 +56,7 @@ public void init(FunctionInvokeContext context) throws IOException { Class udfclass = loader.loadClass(udfDesc.getName()); evalMethod = getEvaluateMethod(functionDesc.getParamTypes(), udfclass); } catch (ClassNotFoundException e) { - e.printStackTrace(); + throw new TajoInternalError(e); } } From a97a29908d6848958cb60d67003766a14e3aae11 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 7 Mar 2016 15:33:39 +0900 Subject: [PATCH 56/72] Apply Optional --- .../apache/tajo/engine/eval/ExprTestBase.java | 2 +- .../engine/function/TestFunctionLoader.java | 42 --------------- .../tajo/engine/function/FunctionLoader.java | 52 +------------------ .../function/hiveudf/HiveFunctionLoader.java | 6 +-- .../org/apache/tajo/master/TajoMaster.java | 33 ++++++------ 5 files changed, 23 insertions(+), 112 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index e52d3a074a..8f7087a09a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -102,7 +102,7 @@ public static void setUp() throws Exception { cat.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); Map map = FunctionLoader.loadBuiltinFunctions(); List list = new ArrayList<>(map.values()); - list.addAll(FunctionLoader.loadUserDefinedFunctions(conf)); + list.addAll(FunctionLoader.loadUserDefinedFunctions(conf).orElse(new ArrayList<>())); for (FunctionDesc funcDesc : list) { cat.createFunction(funcDesc); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java index 26f32626fd..5f1ed681f9 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java @@ -41,7 +41,6 @@ public class TestFunctionLoader { - @Test public void testFindScalarFunctions() throws IOException { List collections = Lists.newArrayList(FunctionLoader.findScalarFunctions()); @@ -51,45 +50,4 @@ public void testFindScalarFunctions() throws IOException { String result = getResultText(TestFunctionLoader.class, "testFindScalarFunctions.result"); assertEquals(result.trim(), functionList.trim()); } - - @Test - public void testAmbiguousException() { - FunctionSignature signature = new FunctionSignature(CatalogProtos.FunctionType.GENERAL, "test1", - CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), - CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT8)); - - FunctionInvocation invocation = new FunctionInvocation(); - FunctionSupplement supplement = new FunctionSupplement(); - - FunctionDesc desc = new FunctionDesc(signature, invocation, supplement); - - List builtins = new ArrayList<>(); - builtins.add(desc); - - signature = new FunctionSignature(CatalogProtos.FunctionType.GENERAL, "test2", - CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT8), - CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT8)); - - desc = new FunctionDesc(signature, invocation, supplement); - builtins.add(desc); - - List udfs = new ArrayList<>(); - - signature = new FunctionSignature(CatalogProtos.FunctionType.UDF, "test1", - CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), - CatalogUtil.newSimpleDataType(TajoDataTypes.Type.FLOAT8)); - - desc = new FunctionDesc(signature, invocation, supplement); - udfs.add(desc); - - boolean afexOccurs = false; - - try { - FunctionLoader.mergeFunctionLists(builtins, udfs); - } catch (AmbiguousFunctionException e) { - afexOccurs = true; - } - - assertTrue(afexOccurs); - } } \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java index 89e85719a4..c309c64470 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java @@ -36,7 +36,6 @@ import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamOptionTypes; import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.exception.AmbiguousFunctionException; import org.apache.tajo.function.*; import org.apache.tajo.plan.function.python.PythonScriptEngine; import org.apache.tajo.util.ClassUtil; @@ -89,7 +88,7 @@ public static Map loadBuiltinFunctions() { * @return * @throws IOException */ - public static List loadUserDefinedFunctions(TajoConf conf) + public static Optional> loadUserDefinedFunctions(TajoConf conf) throws IOException { List functionList = new LinkedList<>(); @@ -122,7 +121,7 @@ public static List loadUserDefinedFunctions(TajoConf conf) } } - return functionList; + return Optional.of(functionList); } public static Set findScalarFunctions() { @@ -284,51 +283,4 @@ public static List findLegacyFunctions() { return sqlFuncs; } - - public static Collection loadFunctions(TajoConf conf) throws IOException, AmbiguousFunctionException { - List functionList = new ArrayList<>(loadBuiltinFunctions().values()); - List udfs = loadUserDefinedFunctions(conf); - - /* NOTE: - * For built-in functions, it is not done to check duplicates. - * There are two reasons. - * Firstly, it could be an useless operation in most of cases because built-in functions are not changed frequently - * but checking will be done each startup. - * Secondly, this logic checks duplicate excluding type, but there are already duplicates in built-in functions - * such as sum with/without 'distinct' feature. So to check duplicates in built-in functions, some other logic is needed. - * It should be another issue. - */ - // merge lists and return it. - return mergeFunctionLists(functionList, udfs); - } - - @SafeVarargs - static Collection mergeFunctionLists(List ... functionLists) - throws AmbiguousFunctionException { - - Map funcMap = new HashMap<>(); - List baseFuncList = functionLists[0]; - - // Build a map with a first list - for (FunctionDesc desc: baseFuncList) { - funcMap.put(desc.hashCodeWithoutType(), desc); - } - - // Check duplicates for other function lists(should be UDFs practically) - for (int i=1; i loadHiveUDFs(TajoConf conf) { + public static Optional> loadHiveUDFs(TajoConf conf) { ArrayList funcList = new ArrayList<>(); String udfdir = conf.getVar(TajoConf.ConfVars.HIVE_UDF_DIR); @@ -52,7 +52,7 @@ public static Collection loadHiveUDFs(TajoConf conf) { FileSystem fs = udfPath.getFileSystem(conf); if (!fs.isDirectory(udfPath)) { - return null; + return Optional.empty(); } // loop each jar file @@ -70,7 +70,7 @@ public static Collection loadHiveUDFs(TajoConf conf) { throw new TajoInternalError(e); } - return funcList; + return Optional.of(funcList); } private static Set> getSubclassesFromJarEntry(URL[] urls, Class targetCls) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index 4e2dc56d0f..3923e32782 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -95,6 +95,7 @@ public class TajoMaster extends CompositeService { @SuppressWarnings("OctalInteger") final public static FsPermission SYSTEM_DIR_PERMISSION = FsPermission.createImmutable((short) 0755); /** rw-r--r-- */ + @SuppressWarnings("OctalInteger") final public static FsPermission SYSTEM_RESOURCE_DIR_PERMISSION = FsPermission.createImmutable((short) 0755); /** rw-r--r-- */ @SuppressWarnings("OctalInteger") @@ -112,9 +113,6 @@ public class TajoMaster extends CompositeService { private FileSystem defaultFS; private Clock clock; - private Path tajoRootPath; - private Path wareHousePath; - private CatalogServer catalogServer; private CatalogService catalog; private GlobalEngine globalEngine; @@ -220,8 +218,6 @@ public void serviceInit(Configuration conf) throws Exception { private Collection loadFunctions() throws IOException, AmbiguousFunctionException { List functionList = new ArrayList<>(FunctionLoader.loadBuiltinFunctions().values()); - Collection udfs = FunctionLoader.loadUserDefinedFunctions(systemConf); - Collection hiveUDFs = HiveFunctionLoader.loadHiveUDFs(systemConf); HashMap funcSet = new HashMap<>(); @@ -229,24 +225,29 @@ private Collection loadFunctions() throws IOException, AmbiguousFu funcSet.put(desc.hashCodeWithoutType(), desc); } - if (udfs != null) { - checkUDFduplicates(udfs, funcSet, functionList); - } - - if (hiveUDFs != null) { - checkUDFduplicates(hiveUDFs, funcSet, functionList); - } + checkUDFduplicateAndMerge(FunctionLoader.loadUserDefinedFunctions(systemConf), funcSet, functionList); + checkUDFduplicateAndMerge(HiveFunctionLoader.loadHiveUDFs(systemConf), funcSet, functionList); return functionList; } - private void checkUDFduplicates(Collection udfs, HashMap funcSet, List funcList) + /** + * Checks duplicates between pre-loaded functions and UDFs. And they are meged to funcList. + * + * @param udfs UDF list + * @param funcSet set for pre-loaded functions to match signature + * @param funcList list to be merged + * @throws AmbiguousFunctionException + */ + private void checkUDFduplicateAndMerge(Optional> udfs, HashMap funcSet, List funcList) throws AmbiguousFunctionException { - for (FunctionDesc desc: udfs) { + for (FunctionDesc desc: udfs.orElse(new ArrayList<>())) { + // check if (funcSet.containsKey(desc.hashCodeWithoutType())) { throw new AmbiguousFunctionException(String.format("UDF %s", desc.toString())); } + // merge funcSet.put(desc.hashCodeWithoutType(), desc); funcList.add(desc); } @@ -277,7 +278,7 @@ private void initWebServer() throws Exception { private void checkAndInitializeSystemDirectories() throws IOException { // Get Tajo root dir - this.tajoRootPath = TajoConf.getTajoRootDir(systemConf); + Path tajoRootPath = TajoConf.getTajoRootDir(systemConf); LOG.info("Tajo Root Directory: " + tajoRootPath); // Check and Create Tajo root dir @@ -302,7 +303,7 @@ private void checkAndInitializeSystemDirectories() throws IOException { } // Get Warehouse dir - this.wareHousePath = TajoConf.getWarehouseDir(systemConf); + Path wareHousePath = TajoConf.getWarehouseDir(systemConf); LOG.info("Tajo Warehouse dir: " + wareHousePath); // Check and Create Warehouse dir From ecaeb225b8414c6225a007d7687672d2b00f7884 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 7 Mar 2016 17:03:39 +0900 Subject: [PATCH 57/72] Some negative test added --- .../tajo/util/TestWritableTypeConverter.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java b/tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java index 0d9378ab56..f0e166bfa4 100644 --- a/tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java +++ b/tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java @@ -23,11 +23,18 @@ import org.apache.hadoop.io.Writable; import org.apache.tajo.datum.DateDatum; import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.Inet4Datum; import org.apache.tajo.datum.TimestampDatum; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; import org.junit.Test; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + import static org.junit.Assert.*; public class TestWritableTypeConverter { @@ -62,4 +69,37 @@ public void testTimestampTypeConverting() { Datum resultDatum = WritableTypeConverter.convertWritable2Datum(resultWritable); assertEquals(testDatum, resultDatum); } + + private static class DummyWritable implements Writable { + @Override + public void write(DataOutput dataOutput) throws IOException { + + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + + } + } + + @Test + public void testNonExistingType() { + try { + WritableTypeConverter.convertWritableToTajoType(DummyWritable.class); + } catch (Exception e) { + assertEquals(TajoRuntimeException.class, e.getClass()); + + TajoRuntimeException runtimeException = (TajoRuntimeException)e; + assertEquals(NotImplementedException.class, runtimeException.getCause().getClass()); + } + + try { + WritableTypeConverter.convertDatum2Writable(new Inet4Datum("11.11.11.11")); + } catch (Exception e) { + assertEquals(TajoRuntimeException.class, e.getClass()); + + TajoRuntimeException runtimeException = (TajoRuntimeException)e; + assertEquals(NotImplementedException.class, runtimeException.getCause().getClass()); + } + } } From 836847a6b838b5d4c43d5413293780679711839f Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Fri, 11 Mar 2016 12:43:13 +0900 Subject: [PATCH 58/72] WritableTypeConverter is moved to tajo-plan --- tajo-common/pom.xml | 6 ------ .../tajo/engine/function/hiveudf/HiveFunctionLoader.java | 2 +- tajo-plan/pom.xml | 6 ++++++ .../org/apache/tajo/plan/function/HiveFunctionInvoke.java | 3 +-- .../org/apache/tajo/plan}/util/WritableTypeConverter.java | 3 +-- .../apache/tajo/plan}/util/TestWritableTypeConverter.java | 2 +- 6 files changed, 10 insertions(+), 12 deletions(-) rename {tajo-common/src/main/java/org/apache/tajo => tajo-plan/src/main/java/org/apache/tajo/plan}/util/WritableTypeConverter.java (98%) rename {tajo-common/src/test/java/org/apache/tajo => tajo-plan/src/test/java/org/apache/tajo/plan}/util/TestWritableTypeConverter.java (99%) diff --git a/tajo-common/pom.xml b/tajo-common/pom.xml index b999a8b5c4..cf4eb36deb 100644 --- a/tajo-common/pom.xml +++ b/tajo-common/pom.xml @@ -249,12 +249,6 @@ powermock-api-mockito test - - org.apache.hive - hive-exec - ${hive.version} - provided - diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index 432558f47a..0397649e69 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -32,7 +32,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.function.UDFInvocationDesc; -import org.apache.tajo.util.WritableTypeConverter; +import org.apache.tajo.plan.util.WritableTypeConverter; import org.reflections.Reflections; import org.reflections.util.ConfigurationBuilder; diff --git a/tajo-plan/pom.xml b/tajo-plan/pom.xml index 32a6df07b0..5c90d69e1c 100644 --- a/tajo-plan/pom.xml +++ b/tajo-plan/pom.xml @@ -172,6 +172,12 @@ org.apache.hadoop hadoop-common + + org.apache.hive + hive-exec + ${hive.version} + provided + junit junit diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java index 89410999e8..01395000a0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java @@ -23,10 +23,9 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.exception.TajoInternalError; -import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.function.UDFInvocationDesc; import org.apache.tajo.storage.Tuple; -import org.apache.tajo.util.WritableTypeConverter; +import org.apache.tajo.plan.util.WritableTypeConverter; import java.io.IOException; import java.lang.reflect.Constructor; diff --git a/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java similarity index 98% rename from tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java index 80fd77f965..94f1f9f1ec 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/WritableTypeConverter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.util; +package org.apache.tajo.plan.util; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; @@ -31,7 +31,6 @@ import org.apache.tajo.util.datetime.DateTimeUtil; import org.reflections.ReflectionUtils; -import java.sql.Timestamp; import java.util.Set; public class WritableTypeConverter { diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java b/tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java similarity index 99% rename from tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java rename to tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java index f0e166bfa4..c799c1cdf4 100644 --- a/tajo-common/src/test/java/org/apache/tajo/util/TestWritableTypeConverter.java +++ b/tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.util; +package org.apache.tajo.plan.util; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; From 37573c0f99d19a63a5f3be134fa026d9be707f85 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Fri, 11 Mar 2016 14:36:16 +0900 Subject: [PATCH 59/72] warning log --- .../tajo/engine/function/hiveudf/HiveFunctionLoader.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index 0397649e69..b7713650a1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -18,6 +18,8 @@ package org.apache.tajo.engine.function.hiveudf; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -43,6 +45,8 @@ import java.util.*; public class HiveFunctionLoader { + private static final Log LOG = LogFactory.getLog(HiveFunctionLoader.class); + public static Optional> loadHiveUDFs(TajoConf conf) { ArrayList funcList = new ArrayList<>(); String udfdir = conf.getVar(TajoConf.ConfVars.HIVE_UDF_DIR); @@ -52,6 +56,7 @@ public static Optional> loadHiveUDFs(TajoConf conf) { FileSystem fs = udfPath.getFileSystem(conf); if (!fs.isDirectory(udfPath)) { + LOG.warn("Hive UDF directory doesn't exist"); return Optional.empty(); } From 1de799b145b34ceaa7586d13a8ad9b1b7f4687ae Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Sat, 12 Mar 2016 00:44:02 +0900 Subject: [PATCH 60/72] Char type added --- .../tajo/plan/util/WritableTypeConverter.java | 13 ++++++++++++- .../tajo/plan/util/TestWritableTypeConverter.java | 11 ++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java index 94f1f9f1ec..48c7bb933c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java @@ -18,7 +18,9 @@ package org.apache.tajo.plan.util; +import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.io.*; import org.apache.tajo.common.TajoDataTypes.Type; @@ -35,11 +37,12 @@ public class WritableTypeConverter { + private static DataType.Builder builder = DataType.newBuilder(); + public static DataType convertWritableToTajoType(Class writableClass) { if (writableClass == null) return null; - DataType.Builder builder = DataType.newBuilder(); Set> parents = ReflectionUtils.getAllSuperTypes(writableClass); if (writableClass == ByteWritable.class || parents.contains(ByteWritable.class)) { @@ -54,6 +57,9 @@ public static DataType convertWritableToTajoType(Class writa if (writableClass == LongWritable.class || parents.contains(LongWritable.class)) { return builder.setType(Type.INT8).build(); } + if (writableClass == HiveCharWritable.class || parents.contains(HiveCharWritable.class)) { + return builder.setType(Type.CHAR).build(); + } if (writableClass == Text.class || parents.contains(Text.class)) { return builder.setType(Type.TEXT).build(); } @@ -95,6 +101,10 @@ public static Writable convertDatum2Writable(Datum value) { result.setTime(DateTimeUtil.julianTimeToJavaTime(value.asInt8())); return result; + case CHAR: { + String str = value.asChars(); + return new HiveCharWritable(new HiveChar(str, str.length())); + } case TEXT: return new Text(value.asChars()); case VARBINARY: return new BytesWritable(value.asByteArray()); } @@ -118,6 +128,7 @@ public static Datum convertWritable2Datum(Writable value) { case TIMESTAMP: return new TimestampDatum(DateTimeUtil.javaTimeToJulianTime( ((TimestampWritable)value).getTimestamp().getTime())); + case CHAR: return new CharDatum(value.toString()); case TEXT: return new TextDatum(value.toString()); case VARBINARY: return new BlobDatum(((BytesWritable)value).getBytes()); } diff --git a/tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java b/tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java index c799c1cdf4..25f99abcf2 100644 --- a/tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java +++ b/tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java @@ -71,15 +71,8 @@ public void testTimestampTypeConverting() { } private static class DummyWritable implements Writable { - @Override - public void write(DataOutput dataOutput) throws IOException { - - } - - @Override - public void readFields(DataInput dataInput) throws IOException { - - } + @Override public void write(DataOutput dataOutput) throws IOException {} + @Override public void readFields(DataInput dataInput) throws IOException {} } @Test From 5c643b309aa3c8afd0b34cab106b1079756e74e2 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 21 Mar 2016 16:11:24 +0900 Subject: [PATCH 61/72] HIVE_UDF_DIR conf name changed --- tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 1828b3d110..9511e5d978 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -310,7 +310,7 @@ public static enum ConfVars implements ConfigKey { PYTHON_CONTROLLER_LOG_DIR("tajo.function.python.controller.log-dir", ""), // HIVE UDF - HIVE_UDF_DIR("hive.udf.dir", "./lib/hiveudf"), + HIVE_UDF_DIR("tajo.function.hive.code-dir", "./lib/hiveudf"), // Partition PARTITION_DYNAMIC_BULK_INSERT_BATCH_SIZE("tajo.partition.dynamic.bulk-insert.batch-size", 1000), From 95f21141c111f4b282d0b4fdec6b3a9dea91d54d Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 21 Mar 2016 16:43:33 +0900 Subject: [PATCH 62/72] Separate HiveFunctionLoader test and just Hive function test. --- .../apache/tajo/engine/eval/ExprTestBase.java | 14 ++ .../hiveudf/TestHiveFunctionLoader.java | 134 +++--------------- .../function/hiveudf/TestHiveFunctions.java | 84 +++++++++++ 3 files changed, 115 insertions(+), 117 deletions(-) create mode 100644 tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 8f7087a09a..39814a67b5 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.eval; +import com.google.common.base.Preconditions; import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.OverridableConf; import org.apache.tajo.SessionVars; @@ -33,6 +34,7 @@ import org.apache.tajo.engine.codegen.EvalCodeGenerator; import org.apache.tajo.engine.codegen.TajoClassLoader; import org.apache.tajo.engine.function.FunctionLoader; +import org.apache.tajo.engine.function.hiveudf.HiveFunctionLoader; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.TajoException; @@ -66,6 +68,7 @@ import java.io.File; import java.io.IOException; +import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -103,6 +106,13 @@ public static void setUp() throws Exception { Map map = FunctionLoader.loadBuiltinFunctions(); List list = new ArrayList<>(map.values()); list.addAll(FunctionLoader.loadUserDefinedFunctions(conf).orElse(new ArrayList<>())); + + // load Hive UDFs + URL hiveUDFURL = ClassLoader.getSystemResource("hiveudf"); + Preconditions.checkNotNull(hiveUDFURL, "hive udf directory is absent."); + conf.set(TajoConf.ConfVars.HIVE_UDF_DIR.varname, hiveUDFURL.toString().substring("file:".length())); + list.addAll(HiveFunctionLoader.loadHiveUDFs(conf).orElse(new ArrayList<>())); + for (FunctionDesc funcDesc : list) { cat.createFunction(funcDesc); } @@ -129,6 +139,10 @@ public TajoConf getConf() { return new TajoConf(conf); } + protected TajoTestingCluster getCluster() { + return cluster; + } + /** * verify query syntax and get raw targets. * diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java index 09bf53e896..ef5d3fef98 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctionLoader.java @@ -1,64 +1,33 @@ -/*** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.tajo.engine.function.hiveudf; -import com.google.common.base.Preconditions; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.FunctionDesc; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.client.TajoClient; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.conf.TajoConf; -import org.junit.After; -import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; - -import java.net.URL; -import java.sql.ResultSet; import java.util.*; -public class TestHiveFunctionLoader { - private TajoTestingCluster cluster; - - @Before - public final void setUp() throws Exception { - cluster = new TajoTestingCluster(); - - TajoConf conf = cluster.getConfiguration(); - - URL hiveUDFURL = ClassLoader.getSystemResource("hiveudf"); - Preconditions.checkNotNull(hiveUDFURL, "hive udf directory is absent."); - conf.set(TajoConf.ConfVars.HIVE_UDF_DIR.varname, hiveUDFURL.toString().substring("file:".length())); - - cluster.startMiniClusterInLocal(1); - } - - @After - public final void tearDown() { - cluster.shutdownCatalogCluster(); - } +import static org.junit.Assert.assertEquals; +public class TestHiveFunctionLoader { @Test public void testAnalyzeUDFclass() { Set> funcSet = new HashSet<>(); @@ -77,73 +46,4 @@ public void testAnalyzeUDFclass() { assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[1].getType()); } - - @Test - public void testFindFunction() throws Exception { - CatalogService catService = cluster.getMaster().getCatalog(); - - FunctionDesc desc = catService.getFunction("my_upper", CatalogProtos.FunctionType.UDF, - CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); - - assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); - assertEquals(1, desc.getParamTypes().length); - assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); - assertEquals("to uppercase", desc.getDescription()); - - TajoDataTypes.DataType int4type = CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4); - desc = catService.getFunction("my_divide", CatalogProtos.FunctionType.UDF, int4type, int4type); - - assertEquals(TajoDataTypes.Type.FLOAT8, desc.getReturnType().getType()); - assertEquals(2, desc.getParamTypes().length); - assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[0].getType()); - assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[1].getType()); - - // synonym - desc = catService.getFunction("test_upper", CatalogProtos.FunctionType.UDF, - CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); - - assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); - assertEquals(1, desc.getParamTypes().length); - assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); - assertEquals("to uppercase", desc.getDescription()); - - // Test for UDF without @Description and including multi 'evaluate()' - desc = catService.getFunction("com_example_hive_udf_MyLower", CatalogProtos.FunctionType.UDF, - CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); - - assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); - assertEquals(1, desc.getParamTypes().length); - assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); - - // same function for another parameter signature - desc = catService.getFunction("com_example_hive_udf_MyLower", CatalogProtos.FunctionType.UDF, - CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); - - assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); - assertEquals(2, desc.getParamTypes().length); - assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); - assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[1].getType()); - } - - @Test - public void testRunFunctions() throws Exception { - TajoClient client = cluster.newTajoClient(); - - ResultSet rs = client.executeQueryAndGetResult("select my_upper('abcd')"); - rs.beforeFirst(); - rs.next(); - String result = rs.getString(1); - - assertEquals("ABCD", result); - - rs.close(); - - rs = client.executeQueryAndGetResult("select my_divide(1,2)"); - rs.beforeFirst(); - rs.next(); - double res = rs.getDouble(1); - assertEquals(0.5, res, 0.0000001); - - rs.close(); - } -} \ No newline at end of file +} diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java new file mode 100644 index 0000000000..6e2971dac8 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java @@ -0,0 +1,84 @@ +/*** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.hiveudf; + +import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.engine.eval.ExprTestBase; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestHiveFunctions extends ExprTestBase { + @Test + public void testFindFunction() throws Exception { + CatalogService catService = getCluster().getCatalogService(); + + FunctionDesc desc = catService.getFunction("my_upper", CatalogProtos.FunctionType.UDF, + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); + + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); + assertEquals(1, desc.getParamTypes().length); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); + assertEquals("to uppercase", desc.getDescription()); + + TajoDataTypes.DataType int4type = CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4); + desc = catService.getFunction("my_divide", CatalogProtos.FunctionType.UDF, int4type, int4type); + + assertEquals(TajoDataTypes.Type.FLOAT8, desc.getReturnType().getType()); + assertEquals(2, desc.getParamTypes().length); + assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[0].getType()); + assertEquals(TajoDataTypes.Type.INT4, desc.getParamTypes()[1].getType()); + + // synonym + desc = catService.getFunction("test_upper", CatalogProtos.FunctionType.UDF, + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); + + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); + assertEquals(1, desc.getParamTypes().length); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); + assertEquals("to uppercase", desc.getDescription()); + + // Test for UDF without @Description and including multi 'evaluate()' + desc = catService.getFunction("com_example_hive_udf_MyLower", CatalogProtos.FunctionType.UDF, + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); + + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); + assertEquals(1, desc.getParamTypes().length); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); + + // same function for another parameter signature + desc = catService.getFunction("com_example_hive_udf_MyLower", CatalogProtos.FunctionType.UDF, + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT)); + + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); + assertEquals(2, desc.getParamTypes().length); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); + assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[1].getType()); + } + + @Test + public void testRunFunctions() throws Exception { + testSimpleEval("select my_upper('abcd')", new String [] {"ABCD"}); + testSimpleEval("select my_divide(1,2)", new String [] {"0.5"}); + } +} \ No newline at end of file From 87cd83703f9226c38b0b449c89d8426a85f34ac1 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 22 Mar 2016 12:23:03 +0900 Subject: [PATCH 63/72] Loop changed to stream --- .../function/hiveudf/HiveFunctionLoader.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index b7713650a1..3b25f22f84 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -20,7 +20,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.exec.Description; @@ -40,6 +39,7 @@ import java.io.IOException; import java.lang.reflect.Method; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.util.*; @@ -60,17 +60,21 @@ public static Optional> loadHiveUDFs(TajoConf conf) { return Optional.empty(); } - // loop each jar file - for (FileStatus fstatus : fs.listStatus(udfPath, (Path path) -> path.getName().endsWith(".jar"))) { + URL [] urls = Arrays.stream(fs.listStatus(udfPath, (Path path) -> path.getName().endsWith(".jar"))) + .map(fstatus -> { + try { + return new URL("jar:" + fstatus.getPath().toUri().toURL() + "!/"); + } catch (MalformedURLException e) { + e.printStackTrace(); + } - URL[] urls = new URL[]{new URL("jar:" + fstatus.getPath().toUri().toURL() + "!/")}; + return null; + }) + .toArray(URL[]::new); + + Set> udfClasses = getSubclassesFromJarEntry(urls, UDF.class); + buildFunctionsFromUDF(udfClasses, funcList, "jar:"+urls[0].getPath()); - // extract and register UDF's decendants (legacy Hive UDF form) - Set> udfClasses = getSubclassesFromJarEntry(urls, UDF.class); - if (udfClasses != null) { - buildFunctionsFromUDF(udfClasses, funcList, "jar:"+urls[0].getPath()); - } - } } catch (IOException e) { throw new TajoInternalError(e); } From c49a7caa1be057a09fc833f7e3fc4ea40e373bf9 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 22 Mar 2016 12:35:15 +0900 Subject: [PATCH 64/72] Add comments --- .../apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index 3b25f22f84..558a2f45e1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -60,6 +60,7 @@ public static Optional> loadHiveUDFs(TajoConf conf) { return Optional.empty(); } + // Read jar paths from the directory and change to URLs URL [] urls = Arrays.stream(fs.listStatus(udfPath, (Path path) -> path.getName().endsWith(".jar"))) .map(fstatus -> { try { @@ -72,6 +73,7 @@ public static Optional> loadHiveUDFs(TajoConf conf) { }) .toArray(URL[]::new); + // Extract UDF classes and build function information Set> udfClasses = getSubclassesFromJarEntry(urls, UDF.class); buildFunctionsFromUDF(udfClasses, funcList, "jar:"+urls[0].getPath()); From 569faeb961469bcab29b2202fa9fdc7b8fba2a79 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 22 Mar 2016 14:54:23 +0900 Subject: [PATCH 65/72] Modify parameter type --- .../main/java/org/apache/tajo/master/TajoMaster.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index 3923e32782..861982d7d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -225,8 +225,8 @@ private Collection loadFunctions() throws IOException, AmbiguousFu funcSet.put(desc.hashCodeWithoutType(), desc); } - checkUDFduplicateAndMerge(FunctionLoader.loadUserDefinedFunctions(systemConf), funcSet, functionList); - checkUDFduplicateAndMerge(HiveFunctionLoader.loadHiveUDFs(systemConf), funcSet, functionList); + checkUDFduplicateAndMerge(FunctionLoader.loadUserDefinedFunctions(systemConf).orElse(Collections.emptyList()), funcSet, functionList); + checkUDFduplicateAndMerge(HiveFunctionLoader.loadHiveUDFs(systemConf).orElse(Collections.emptyList()), funcSet, functionList); return functionList; } @@ -239,9 +239,9 @@ private Collection loadFunctions() throws IOException, AmbiguousFu * @param funcList list to be merged * @throws AmbiguousFunctionException */ - private void checkUDFduplicateAndMerge(Optional> udfs, HashMap funcSet, List funcList) + private static void checkUDFduplicateAndMerge(List udfs, HashMap funcSet, List funcList) throws AmbiguousFunctionException { - for (FunctionDesc desc: udfs.orElse(new ArrayList<>())) { + for (FunctionDesc desc: udfs) { // check if (funcSet.containsKey(desc.hashCodeWithoutType())) { throw new AmbiguousFunctionException(String.format("UDF %s", desc.toString())); @@ -557,7 +557,7 @@ public TajoRestService getRestServer() { } } - String getThreadTaskName(long id, String name) { + private String getThreadTaskName(long id, String name) { if (name == null) { return Long.toString(id); } From f2c414d5a67cd0b13e1690e1837feae0efafe3a2 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 22 Mar 2016 15:23:39 +0900 Subject: [PATCH 66/72] Throw exception when Hive UDF is not found. --- .../tajo/plan/function/HiveFunctionInvoke.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java index 01395000a0..9090da3b1e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java @@ -22,7 +22,9 @@ import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; +import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.exception.UndefinedFunctionException; import org.apache.tajo.function.UDFInvocationDesc; import org.apache.tajo.storage.Tuple; import org.apache.tajo.plan.util.WritableTypeConverter; @@ -33,6 +35,7 @@ import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; +import java.util.Arrays; public class HiveFunctionInvoke extends FunctionInvoke implements Cloneable { private Object instance = null; @@ -59,7 +62,7 @@ public void init(FunctionInvokeContext context) throws IOException { } } - private Method getEvaluateMethod(TajoDataTypes.DataType [] paramTypes, Class clazz) { + private Method getEvaluateMethod(TajoDataTypes.DataType [] tajoParamTypes, Class clazz) { Constructor constructor = clazz.getConstructors()[0]; try { @@ -71,16 +74,20 @@ private Method getEvaluateMethod(TajoDataTypes.DataType [] paramTypes, Class for (Method m: clazz.getMethods()) { if (m.getName().equals("evaluate")) { Class [] methodParamTypes = m.getParameterTypes(); - if (checkParamTypes(methodParamTypes, paramTypes)) { + if (checkParamTypes(methodParamTypes, tajoParamTypes)) { return m; } + else { + throw new TajoInternalError(new UndefinedFunctionException( + String.format("Hive UDF (%s)", clazz.getSimpleName()))); + } } } return null; } - boolean checkParamTypes(Class [] writableParams, TajoDataTypes.DataType [] tajoParams) { + private boolean checkParamTypes(Class [] writableParams, TajoDataTypes.DataType [] tajoParams) { int i = 0; for (Class writable: writableParams) { From b552112ff919a1e282e16ddb0ab4642bfb7ff64b Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Tue, 22 Mar 2016 15:24:39 +0900 Subject: [PATCH 67/72] Code error is fixed --- .../org/apache/tajo/plan/function/HiveFunctionInvoke.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java index 9090da3b1e..37be0de3de 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java @@ -77,14 +77,10 @@ private Method getEvaluateMethod(TajoDataTypes.DataType [] tajoParamTypes, Class if (checkParamTypes(methodParamTypes, tajoParamTypes)) { return m; } - else { - throw new TajoInternalError(new UndefinedFunctionException( - String.format("Hive UDF (%s)", clazz.getSimpleName()))); - } } } - - return null; + + throw new TajoInternalError(new UndefinedFunctionException(String.format("Hive UDF (%s)", clazz.getSimpleName()))); } private boolean checkParamTypes(Class [] writableParams, TajoDataTypes.DataType [] tajoParams) { From 8758a878ea888c3d4ded5e5282ce84b182330ece Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Thu, 21 Apr 2016 18:33:16 +0900 Subject: [PATCH 68/72] Warning message refined --- .../apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index 558a2f45e1..e498d667b1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -56,7 +56,7 @@ public static Optional> loadHiveUDFs(TajoConf conf) { FileSystem fs = udfPath.getFileSystem(conf); if (!fs.isDirectory(udfPath)) { - LOG.warn("Hive UDF directory doesn't exist"); + LOG.warn("Hive UDF directory doesn't exist : "+udfdir); return Optional.empty(); } From 089935e62f98de0fe60775c64280cfce26bfb701 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 25 Apr 2016 13:47:11 +0900 Subject: [PATCH 69/72] Filesystem problem fixed --- .../tajo/engine/function/hiveudf/HiveFunctionLoader.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index e498d667b1..c5a764e32a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -52,8 +52,9 @@ public static Optional> loadHiveUDFs(TajoConf conf) { String udfdir = conf.getVar(TajoConf.ConfVars.HIVE_UDF_DIR); try { + // Currently Hive udf jar must be on local filesystem + FileSystem fs = FileSystem.getLocal(conf); Path udfPath = new Path(udfdir); - FileSystem fs = udfPath.getFileSystem(conf); if (!fs.isDirectory(udfPath)) { LOG.warn("Hive UDF directory doesn't exist : "+udfdir); From 2f756ea874a15b15fb1e4683b8ab67200217adf9 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 25 Apr 2016 14:08:02 +0900 Subject: [PATCH 70/72] Handling NULL parameter --- .../tajo/engine/function/hiveudf/TestHiveFunctions.java | 1 + .../org/apache/tajo/plan/util/WritableTypeConverter.java | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java index 6e2971dac8..a212a7962d 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java @@ -78,6 +78,7 @@ public void testFindFunction() throws Exception { @Test public void testRunFunctions() throws Exception { + testSimpleEval("select my_upper(null)", new String [] {"NULL"}); testSimpleEval("select my_upper('abcd')", new String [] {"ABCD"}); testSimpleEval("select my_divide(1,2)", new String [] {"0.5"}); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java index 48c7bb933c..4ba0d9861a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java @@ -107,12 +107,18 @@ public static Writable convertDatum2Writable(Datum value) { } case TEXT: return new Text(value.asChars()); case VARBINARY: return new BytesWritable(value.asByteArray()); + + case NULL_TYPE: return null; } throw new TajoRuntimeException(new NotImplementedException(value.type().name())); } public static Datum convertWritable2Datum(Writable value) { + if (value == null) { + return NullDatum.get(); + } + DataType type = convertWritableToTajoType(value.getClass()); switch(type.getType()) { From ab98376243d408cb54e5ef07ad20f7da3b58fba9 Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Wed, 27 Apr 2016 18:12:35 +0900 Subject: [PATCH 71/72] Exception handling changed --- .../function/hiveudf/HiveFunctionLoader.java | 24 ++++++++++++------- .../plan/function/HiveFunctionInvoke.java | 12 ++++++---- .../tajo/plan/util/WritableTypeConverter.java | 6 ++--- .../plan/util/TestWritableTypeConverter.java | 10 ++++---- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java index c5a764e32a..98ae4cd1fd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/hiveudf/HiveFunctionLoader.java @@ -32,6 +32,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.exception.UnsupportedDataTypeException; import org.apache.tajo.function.UDFInvocationDesc; import org.apache.tajo.plan.util.WritableTypeConverter; import org.reflections.Reflections; @@ -145,20 +146,25 @@ static void buildFunctionsFromUDF(Set> classes, List list) { - TajoDataTypes.DataType retType = - WritableTypeConverter.convertWritableToTajoType((Class)method.getReturnType()); - TajoDataTypes.DataType[] params = convertTajoParamterTypes(method.getParameterTypes()); + try { + TajoDataTypes.DataType retType = + WritableTypeConverter.convertWritableToTajoType((Class) method.getReturnType()); + TajoDataTypes.DataType[] params = convertTajoParamterTypes(method.getParameterTypes()); - builder.setReturnType(retType).setParams(params); + builder.setReturnType(retType).setParams(params); - for (String name: names) { - builder.setName(name); - builder.setUDF(udfInvocation); - list.add(builder.build()); + for (String name : names) { + builder.setName(name); + builder.setUDF(udfInvocation); + list.add(builder.build()); + } + } catch (UnsupportedDataTypeException e) { + LOG.error(String.format("Hive UDF '%s' is not registered because of unsupported type: %s", names[0], e.getMessage())); } } - private static TajoDataTypes.DataType[] convertTajoParamterTypes(Class[] hiveUDFparams) { + private static TajoDataTypes.DataType[] convertTajoParamterTypes(Class[] hiveUDFparams) + throws UnsupportedDataTypeException { TajoDataTypes.DataType [] params = null; // convert types to ones of Tajo diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java index 37be0de3de..2435195d87 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/HiveFunctionInvoke.java @@ -22,9 +22,7 @@ import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; -import org.apache.tajo.exception.TajoException; -import org.apache.tajo.exception.TajoInternalError; -import org.apache.tajo.exception.UndefinedFunctionException; +import org.apache.tajo.exception.*; import org.apache.tajo.function.UDFInvocationDesc; import org.apache.tajo.storage.Tuple; import org.apache.tajo.plan.util.WritableTypeConverter; @@ -87,8 +85,12 @@ private boolean checkParamTypes(Class [] writableParams, TajoDataTypes.DataType int i = 0; for (Class writable: writableParams) { - if (!WritableTypeConverter.convertWritableToTajoType(writable).equals(tajoParams[i++])) { - return false; + try { + if (!WritableTypeConverter.convertWritableToTajoType(writable).equals(tajoParams[i++])) { + return false; + } + } catch (UnsupportedDataTypeException e) { + throw new TajoRuntimeException(e); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java index 4ba0d9861a..d83524cd50 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/WritableTypeConverter.java @@ -39,7 +39,7 @@ public class WritableTypeConverter { private static DataType.Builder builder = DataType.newBuilder(); - public static DataType convertWritableToTajoType(Class writableClass) { + public static DataType convertWritableToTajoType(Class writableClass) throws UnsupportedDataTypeException { if (writableClass == null) return null; @@ -79,7 +79,7 @@ public static DataType convertWritableToTajoType(Class writa return builder.setType(Type.VARBINARY).build(); } - throw new TajoRuntimeException(new NotImplementedException(writableClass.getSimpleName())); + throw new UnsupportedDataTypeException(writableClass.getSimpleName()); } public static Writable convertDatum2Writable(Datum value) { @@ -114,7 +114,7 @@ public static Writable convertDatum2Writable(Datum value) { throw new TajoRuntimeException(new NotImplementedException(value.type().name())); } - public static Datum convertWritable2Datum(Writable value) { + public static Datum convertWritable2Datum(Writable value) throws UnsupportedDataTypeException { if (value == null) { return NullDatum.get(); } diff --git a/tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java b/tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java index 25f99abcf2..306cbe3454 100644 --- a/tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java +++ b/tajo-plan/src/test/java/org/apache/tajo/plan/util/TestWritableTypeConverter.java @@ -27,6 +27,7 @@ import org.apache.tajo.datum.TimestampDatum; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedDataTypeException; import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; import org.junit.Test; @@ -39,7 +40,7 @@ public class TestWritableTypeConverter { @Test - public void testDateTypeConverting() { + public void testDateTypeConverting() throws UnsupportedDataTypeException { Datum testDatum; TimeMeta testTM = new TimeMeta(); @@ -57,7 +58,7 @@ public void testDateTypeConverting() { } @Test - public void testTimestampTypeConverting() { + public void testTimestampTypeConverting() throws UnsupportedDataTypeException { Datum testDatum; long currentMills = System.currentTimeMillis(); @@ -80,10 +81,7 @@ public void testNonExistingType() { try { WritableTypeConverter.convertWritableToTajoType(DummyWritable.class); } catch (Exception e) { - assertEquals(TajoRuntimeException.class, e.getClass()); - - TajoRuntimeException runtimeException = (TajoRuntimeException)e; - assertEquals(NotImplementedException.class, runtimeException.getCause().getClass()); + assertEquals(UnsupportedDataTypeException.class, e.getClass()); } try { From c3a217dcb9999f8beeefb7a6759f28aa514de31a Mon Sep 17 00:00:00 2001 From: Jongyoung Park Date: Mon, 2 May 2016 12:41:06 +0900 Subject: [PATCH 72/72] type checking bug fixed --- .../function/hiveudf/TestHiveFunctions.java | 17 +++++++++++++++++ .../src/test/resources/hiveudf/my_udf_src.zip | Bin 3305 -> 5599 bytes .../src/test/resources/hiveudf/myudf.jar | Bin 3129 -> 5519 bytes .../plan/function/HiveFunctionInvoke.java | 4 ++++ 4 files changed, 21 insertions(+) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java index a212a7962d..a2c89fbe29 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/hiveudf/TestHiveFunctions.java @@ -74,6 +74,19 @@ public void testFindFunction() throws Exception { assertEquals(2, desc.getParamTypes().length); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[0].getType()); assertEquals(TajoDataTypes.Type.TEXT, desc.getParamTypes()[1].getType()); + + // multiple signatures + // my_substr has two types, (Text, IntWritable) and (Text, IntWritable, IntWritable) + desc = catService.getFunction("my_substr", CatalogProtos.FunctionType.UDF, + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4), + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4)); + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); + assertEquals(3, desc.getParamTypes().length); + + desc = catService.getFunction("my_substr", CatalogProtos.FunctionType.UDF, + CatalogUtil.newSimpleDataType(TajoDataTypes.Type.TEXT), CatalogUtil.newSimpleDataType(TajoDataTypes.Type.INT4)); + assertEquals(TajoDataTypes.Type.TEXT, desc.getReturnType().getType()); + assertEquals(2, desc.getParamTypes().length); } @Test @@ -81,5 +94,9 @@ public void testRunFunctions() throws Exception { testSimpleEval("select my_upper(null)", new String [] {"NULL"}); testSimpleEval("select my_upper('abcd')", new String [] {"ABCD"}); testSimpleEval("select my_divide(1,2)", new String [] {"0.5"}); + + // my_substr() uses 1-based index + testSimpleEval("select my_substr('abcde', 3)", new String [] {"cde"}); + testSimpleEval("select my_substr('abcde', 1, 2)", new String [] {"ab"}); } } \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/hiveudf/my_udf_src.zip b/tajo-core-tests/src/test/resources/hiveudf/my_udf_src.zip index 1f61a865b0164e1e29da0b3925696945763b388f..62c9b5c5d76209212a6721180278091ee427fd40 100644 GIT binary patch literal 5599 zcmbW4c{tSDAIFE8A?w7*I+iZQ5JpIL3S(a~_9cvMWSK^YCN;7qgshc_WS7Vu;+7>s zC3{Lj*PetnznS!Vj7z0Ef6Vec^L@R~d7pDWpYt}<0fJ}&q}RNy2=e=vA8#N44B(AJ zi<_Wm0Km*6?@fdn_nY~grUOubmM8!K5DW2thCh9p=Eu_>_889{B0l|V0F2}aH+x_E zbxy_ahVpL?^A--;%i}+?IJadvowoPDx;yLQ9$oy&upBMez9)dZN_qRqaBeQEPiy@nPsRc<4p`>c zP8M%Fg-M?9G0i`BMqSVv%rL0MmoAX_wi%x;DKW!d<&SK=`P9Hc21g* z6LbT)AsT6Lsbl52YZZ+(IwsLVdJ@Yj46RC!4Bws%;+CVsN}G8DNrz$GtJ1hgWEZs% zayQW@1xf$_u|u!?)iAyoN2d+HZYH>yZsS)}Os7s0f~9$GRfI?J9^$WAOAti8Pu1We zvNf)EwnQodV`=8NR%3CuM9;qODDB8`d>RxdA_u~piI(q{kHQ~?zBjrjjeaga1vDQy z`8s$=%~X%ECOg!n66A*PM`~a)gjbNVmSQFP)2t^BMx~nN5l;W23KUpxVN;+4Ty!?l_rT5k6 zS5V;mp#~kh0-uMpvCdBUGxxkBx1HyN&*k-0(JlqYP2Sk=?iNXd~ zZi~7t&iXJ5Pwox6>Nj!qdI@rgz)is2&w_q@d3k6G*g8}ujjmo66ToJ~Pa4fchDUh^ zIxFcdT#oaKF<3g7cO8GrShyD-C}T-D$j(1qmo^+r$r9lX&43Xkfh(aJbX^pBkJM^= zSUYMX9U1|r=oA{Afrq&G`3&10=pT*XBn*Z|$Hya6hID8Le66ocu+%@)v<|Okr}}u~ z!#@;(F;g{%dI}k>#neB;WhAl3an|&7CixGG@+-eh@d9foH+&?O5)ScHLGyHxKdnYk%E?LA0}o;} z_1lBsjAt#z;gUzjU-@L#K~~!Kv|Nj?Kp~P`6IzCZn3AX<2g7!`=N-PWI5Y|18y#wd zAm7YaQbS`c)=;h!VIbDV_AU)qb4?#QXs`oDSV-!z%GhZ{g<6Zz0^B&aUmM_k6jM)mtMxv|`=-ZpJ_wVn8<^4LWyRrs`5^{@aQ&dbK7)gGU?rxaau zT5b`JElSJHe-37&diPg}e=JR+;ES3z&pkldz+4qaclH6vtF$ex*&K)qFF`8jB+(cB zW`HcmX-a$a%9c}`57<8+M7~IP=AU-5u*7miJLAry1!i+X0W&Lq?a6;OoOMhJ`}N@N2de5m;K;dGvh*W4$G7FtfPczv6Lo`xTo6n>Z(aP|Us-c@OI?v;?mZ;5+34T7LW(0#O{UH;jR zSGCrvvi%eu#L|mRhpi;tRx~qcTM1#u&v2;qP2)KeBz(V`JIC6$eX*~kIG4wBgz83O z=#&mhL~9lE3?b7g30pkVp4si2-^~;h5$C;f2@YU=={EXC<5}&s4q2G0cU^7wi;M`j z=6djf}0dw#}$gdV_^e?NJ~1WvPOKfJqk4_ z5zDbTiV(0*RVdS_gp09#X(&jY4|t0Ti&p>U=xWIaVnJ&WVguLAJ_w&I*xPKO4fi{! zE=d*kSt8SmgCMg^5ez-zf9};N^zL=T+?i@*(o7#*Myj@I(2gc>pT%c(B-9L*XA`rF zj^7MSUpH>Wrzlf1S2yZ8Kp6(vy+n2YjRflLz*K2)^Rbt+_g)o3zLs{wT@xDk6d0PZ zeXBI=cAcDaFy)Y%H3hqGxUmY+!+4%|_p?Vy!`0sm4hZTFpnE^y!5JW(uKE zEPZB&ZA{T)jv1KnS7(4#EZRd7EJJFQbAH}FGh^NTRasS_o)TNe+R!CDDDzr@ptjfq zUF6buHM-NpM`L%IfeVBS!J7Om#{tn8sLlw8Uow}X796wbh4L|VjoKsLN3UINWqC{nxj#s}$86!t5sI2ORUIY#{FO1&$yw~^C2j+*=U>o~T!zY}p9|B7m;AD?QxW+fl_~SMN33$U{@Tga%Wu)2 z{9V6LZ&dVmo>-(yO?K5Pu`?F`NoO>{V#zz>J$tR~JEI<{GlncHJrokL%EJ?cn3?y4 zgAcgYXk3ugDwHcJS^TC-JDeNUOeK>x=ZiA^70Ls0Tnw{?UmW&=CM>Dx*VGo9+v z?X8$r4AHxpoOrh%HF{~knSgB0*^pS%Lfw6X#>$UP-qd`ZdiFTeHQ7$7o~uxW;}TO^ zahSz%ZYFJq6=6wJ*C*6c#f@tCF4m5xkvAOx0RWzisRm~>2&66B_X+OT(O?j!luzoa zVqg!Xn6^dlNYbQrcsaZ0^rJe84Ttg0(a5o}3#v>+caw;>KB=AEJ&aG`FrF?)y*$zO zr#3Dig9@J8*_>dcXxxOLA6^uZ3VjGX!FIT<_c;MQ%8s(qQ{x`&)82*sIycE#X?sSc z_*s0~hYyaj88+ync4!yZzs39yib5O%DpF1M_`#hC$y^s`Jo42~No40v^yG&Y_+J~b z9?zZ=YFToH(=;RGi)J*tGSaJzM_DpvojGZ*^rsHe_=KPvkMws}MLcG^+n6B`lHB2N zb!2Vq)8un;MXNhJIt1!gSZY4|0opDh7Hh(FINlV%7D>Q4BRS7SLb55TSv5L&QM^~u z()P|uzNOFVSSW1p3LBQAbc|D?-gSRja^Inzb7ueylV@1+Cd!_&;-JMZJ!AST=DxOK z(w?=Z?@jU6;u#UVnWm!n_>f_-@*~eYagzPp=T|7X{WvFYFCQG*$$LHR{O34(%E|lG ze?iU^>p2~~p3}Ev^q-SxHj$KmkbY$L z-{bzBv>7+y2;{gQ6DBfA`TZ9|{5O+%6SH*-b3M!dIhS)2cMtK*l1%(Q_3vXPaW``P zpYwS)@iD*T|H$}65mLtgg*jUjKKV{){JqF}-X{u??&aGG{VeUT3lS9nSp;5Ub^Ezn z-)BP-BFp|nMN;i|)dbRGD`RJZm-a#2-=9f@w!A&JVQ4!N!J z_C*uA={znm=BUxK|~{- zq#%ut;#sYFvq}{e4|o7q_0g|J~}e)FIA{$;# zKGJHIQu%^PcF%0iKg@i~f0uN!5{(@uB2+C_YQ6DJdteYn7mAmQR>~`Z=(PJs0vN%Q zO7-diiE@^wr?d9qYE%Rh{tVP0{}s;HdaoQIj9 zLMIA)H|{QHam5&~h|{frsFu22oD(L) zXiSr?FATB19KF5Gx_)Gsu9Ur6!zM)_;*or#QEncGGlRiF(aYitD56>Q52Iat`}oaI zMq9Vhu3fleqfIbDZGF-d^m)3*hkEMKCC#nu?A%KGS^YgW6n(G7M$nZu%dPrSvs7;J z>w^9VSYxr=TIBIaH%@20191W3VapqsF_)7ZeKPfXL!(ZjnpbgZKMkM4kBa3-D>-}v zr;Zz`hCh4rg*SekPuLDbvvsRrgdtRLz<{DJ=5#)(9xUMzedY^I7XDWP1BoYUYyFj# z_R(2~63G8kPpeW`!+nwxeeTm)w&$do4Qqe}_F8e+*6peZ@LK>drvP+!0=s++0oHDQ z5CE?lge93`lFcQ-3=n~Bp0Y7!WIhFBWX{#ESnd|6UOBKA*yDE8s?$InHKo9IU@5Q( z*xwU2DZgFDf%@s19`GCEz_s*y^wnZW4 hwpjV0jNO+%aE0YdbAje^u0vocjlK7sgp4cj?=QJia6|wA diff --git a/tajo-core-tests/src/test/resources/hiveudf/myudf.jar b/tajo-core-tests/src/test/resources/hiveudf/myudf.jar index 84e86e48022d394bcdfffb95d40fc55518147c9e..8124353685d6d74babdb0907e513e74755149e15 100644 GIT binary patch delta 2517 zcmZ9Oc{CK<8^_1i*rw#QPu8(!EEzKtg)}nRN60dUNR}*F6Ph;F$Pgo0v+rw2M5fTp zAX~PvlO;>GOqL=q-gAD(`Q7`+_uS_^=YH?G=Q-!|trdut2bmf$F|z{ztN;KEn;8Rw zFdYa?=1?%uLy1&ADUZ$62OL?VL?0GGoTHnUqSGx$Kff@vk#~QxB`%uP6+O5Mg$t(9r&cHS+PoM{O1YTgSY4hNVgB1qTD-%TfjSC zTepCYfA7AQOSyguox8NC{a8lALb}1QzalXSM&H%GZ1);3%O;gp5I(&<4c6j5Ey67; z(OEtp_RHk+#Z}X0^9!wLUX!Pv^<#|j+2&}M{O>3U^CwFldMP-S07eA*Y-dxXBV}Md z?)fEV7-Iy!pUo->XKdjTFP#s=eGL|(2DRE{xK(Sa5;5vEpUVS!#Q06tzBKyriyTwj zDEFTH_?hFB3cZx~f#dFiyC^wNCH=f-EtjLB%FyIXtA_PAnr%-Wf!+5`Mt6KqjGPk> zFs6^bOnSVdk9LvF=Q#y5%9roNOi0wx85-)UfkHORc$hwXg_Z?&a z0JN9@fP;bI0kE>N0+{xv;tk23@jxrZn6wxVr-dYoGY`1#=kNR9Xp~rvn}Y{<|Hh9Z z#ablAKM^8Oh$vf0=!*g;J12@9nJbR9RwtoZI_iTB#=+CgY7pb74tV^Q)KFm3y!)$(=xquFzMzH2F^A!>eY-(8+Qn+6s~J<29|~f@ey3xLGXGT-yJ<2Pa44`HD)Bh!nMzmeLeZ5B|MT* zC#N6Z66Lz?w!yV?8FIk}666|I^{fC9mf649Iu#^aJNj;5>Eh^yCC5uypAR^T(;J8% zz(QJSn1n5q&(9a&A~48+ag7U@=}Smjgxi{-LP!7Zp*2oz@dMRK)q(}9B-zfr$f-eX zN#|b~PhgaulD$h4(QFFVYqk%vTrlGEcwv#K7C8?pw5p!!nJ=a0UvOb2_@49S(v=vt z?XVq6=-(RSd-8ccm~i6>1P+;co3qnPHluZRRQb5h!{T>J<60{nV^EXvcHcAJG2U&# z&qM2om3k%G$dZp7C9X!+o-mb;pH|Q-c2y90X=_)r%$yq`@HP0~m@GVi!kOJbl#qM=QEKwsgP zIG0@h6621h%Id7bhCK&$Zw%GCl0$8_MA@epeY>H?m4MA{_o+|S90{b0IR%_r{jVPLXT0%UePHoCk0ZF7kyT~lu*tzaM=>!Zm=oI(zbQ8>2X_GGEMDa}f?NnU@N zlq*u;lPH<~&8mgj(CCcdm*XeAAOWjm#S6w)6dX5iGx*Q#P{#;!+J^M`T_gLu7 zNQD%d-gtK6nKb{a&Qt+GbJ4a0GL7%`aO3(KS&@1sTisqN{ax2n4}p)W2EM7tJINEX ztna9)K{YDTCrqP0VLh3C+~=Q%KF|A%ejSf_qk09E*TL+>bL}xoTIn;QwsZI^Y7^Ug z(tG9;u57KbD^0TRla?6VwO2Z_tV?T43a$HS^U`v^Ok@T_R^USPoN$sd>Eg=$+Y4il zTSDLeVwigJg?wMIGi2Lnf|s1}bM6ulahN$?d%VFJdaI(xY2khzR&34n=6OSrYdK#|6w$XCEQZ+L zn&&9mMJbd`v~P(7#`K;WV_yRshSwkK5f{|bB^KQ9KbsJ+x`Eqg50JHjhHlU~1_0aw z9c9tcfI^gG60MA2;rI_s2|(;2n25jGU4ql>2mFt_=ycbZaz~3mi=!40ND3;KO+E; I#eH!47wf8zP5=M^ delta 296 zcmeCz-YLNw;LXg!#lXP9z_6k;Vj{0HGl())2UD>+V5-LeL{08s*VxR