[HIVEMALL-289] Add str_contain(string str, array<string> match, boole…

…an or=true) UDF

## What changes were proposed in this pull request?

Add str_contain(string str, array<string> match, boolean or=true) UDF

## What type of PR is it?

Feature

## What is the Jira issue?

https://issues.apache.org/jira/browse/HIVEMALL-289

## How was this patch tested?

manual tests on EMR

## How to use this feature?

```sql
select
  str_contains('There are apple and orange', array('apple')),
  str_contains('There are apple and orange', array('apple', 'banana'), true),
  str_contains('There are apple and orange', array('apple', 'banana'), false);
> true, true, false
```

## Checklist

- [x] Did you apply source code formatter, i.e., `./bin/format_code.sh`, for your commit?
- [x] Did you run system tests on Hive (or Spark)?

Author: Makoto Yui <myui@apache.org>

Closes #225 from myui/HIVEMALL-289.

ChangeLog.md

@@ -1,3 +1,22 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
 # v0.6.0 - Dec 19, 2019
 
 Major new features in this release includes:

core/src/main/java/hivemall/tools/strings/StrContainsUDF.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.tools.strings;
+
+import hivemall.utils.hadoop.HiveUtils;
+
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+
+//@formatter:off
+@Description(name = "str_contains",
+        value = "_FUNC_(string query, array<string> searchTerms [, boolean orQuery=false])"
+                + " - Returns true if the given query contains search terms",
+        extended = "select\n" + 
+                "  str_contains('There are apple and orange', array('apple')), -- or=false\n" + 
+                "  str_contains('There are apple and orange', array('apple', 'banana'), true), -- or=true\n" + 
+                "  str_contains('There are apple and orange', array('apple', 'banana'), false); -- or=false\n" + 
+                "> true, true, false")
+//@formatter:on
+@UDFType(deterministic = true, stateful = false)
+public final class StrContainsUDF extends GenericUDF {
+
+    private StringObjectInspector queryOI;
+    private ListObjectInspector searchTermsOI;
+    private BooleanObjectInspector orQueryOI;
+
+    @Override
+    public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
+        if (argOIs.length != 2 && argOIs.length != 3) {
+            throw new UDFArgumentLengthException("str_contains expects two or three arguments");
+        }
+
+        this.queryOI = HiveUtils.asStringOI(argOIs, 0);
+        if (!HiveUtils.isStringListOI(argOIs[1])) {
+            throw new UDFArgumentTypeException(1,
+                "Expected array<string> for the second argument but got "
+                        + argOIs[1].getTypeName());
+        }
+        this.searchTermsOI = HiveUtils.asListOI(argOIs, 1);
+
+        if (argOIs.length == 3) {
+            this.orQueryOI = HiveUtils.asBooleanOI(argOIs, 2);
+        }
+
+        return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
+    }
+
+    @Override
+    public Boolean evaluate(DeferredObject[] args) throws HiveException {
+        final String query = queryOI.getPrimitiveJavaObject(args[0].get());
+        if (query == null) {
+            return null;
+        }
+
+        final List<String> searchTerms = HiveUtils.asStringList(args[1], searchTermsOI);
+        if (searchTerms == null || searchTerms.isEmpty()) {
+            return Boolean.FALSE;
+        }
+
+        boolean orQuery = false;
+        if (args.length == 3) {
+            orQuery = orQueryOI.get(args[2].get());
+        }
+
+        if (orQuery) {
+            for (String term : searchTerms) {
+                if (query.contains(term)) {
+                    return Boolean.TRUE;
+                }
+            }
+            return Boolean.FALSE;
+        } else {
+            for (String term : searchTerms) {
+                if (!query.contains(term)) {
+                    return Boolean.FALSE;
+                }
+            }
+            return Boolean.TRUE;
+        }
+    }
+
+    @Override
+    public String getDisplayString(String[] args) {
+        return "str_contains(" + StringUtils.join(args, ',') + ')';
+    }
+
+
+}

core/src/main/java/hivemall/utils/hadoop/HiveUtils.java

@@ -1065,6 +1065,17 @@ public static BooleanObjectInspector asBooleanOI(@Nonnull final ObjectInspector
         return (BooleanObjectInspector) argOI;
     }
 
+    @Nonnull
+    public static BooleanObjectInspector asBooleanOI(@Nonnull final ObjectInspector[] argOIs,
+            final int argIndex) throws UDFArgumentException {
+        ObjectInspector argOI = getObjectInspector(argOIs, argIndex);
+        if (!BOOLEAN_TYPE_NAME.equals(argOI.getTypeName())) {
+            throw new UDFArgumentTypeException(argIndex,
+                "Argument type must be Boolean: " + argOI.getTypeName());
+        }
+        return (BooleanObjectInspector) argOI;
+    }
+
     @Nonnull
     public static IntObjectInspector asIntOI(@Nonnull final ObjectInspector argOI)
             throws UDFArgumentException {

resources/ddl/define-all-as-permanent.hive

@@ -676,6 +676,9 @@ CREATE FUNCTION unbase91 as 'hivemall.tools.text.Unbase91UDF' USING JAR '${hivem
 DROP FUNCTION IF EXISTS word_ngrams;
 CREATE FUNCTION word_ngrams as 'hivemall.tools.text.WordNgramsUDF' USING JAR '${hivemall_jar}';
 
+DROP FUNCTION IF EXISTS str_contains;
+CREATE FUNCTION str_contains as 'hivemall.tools.strings.StrContainsUDF' USING JAR '${hivemall_jar}';
+
 ---------------------------------
 -- Dataset generator functions --
 ---------------------------------

resources/ddl/define-all.hive

@@ -668,6 +668,9 @@ create temporary function unbase91 as 'hivemall.tools.text.Unbase91UDF';
 drop temporary function if exists word_ngrams;
 create temporary function word_ngrams as 'hivemall.tools.text.WordNgramsUDF';
 
+drop temporary function if exists str_contains;
+create temporary function str_contains as 'hivemall.tools.strings.StrContainsUDF';
+
 ---------------------------------
 -- Dataset generator functions --
 ---------------------------------

resources/ddl/define-all.spark

@@ -652,6 +652,9 @@ sqlContext.sql("CREATE TEMPORARY FUNCTION unbase91 AS 'hivemall.tools.text.Unbas
 sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS word_ngrams")
 sqlContext.sql("CREATE TEMPORARY FUNCTION word_ngrams AS 'hivemall.tools.text.WordNgramsUDF'")
 
+sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS str_contains")
+sqlContext.sql("CREATE TEMPORARY FUNCTION str_contains AS 'hivemall.tools.strings.StrContainsUDF'")
+
 /**
  * Dataset generator functions
  */

tools/hivemall-docs/src/main/java/hivemall/docs/FuncsListGeneratorMojo.java

@@ -96,7 +96,7 @@ public class FuncsListGeneratorMojo extends AbstractMojo {
         genericFuncsHeaders.put("# Sanity Checks",
             Collections.singletonList("hivemall.tools.sanity"));
         genericFuncsHeaders.put("# Text processing",
-            Collections.singletonList("hivemall.tools.text"));
+            Arrays.asList("hivemall.tools.text", "hivemall.tools.strings"));
         genericFuncsHeaders.put("# Timeseries",
             Collections.singletonList("hivemall.tools.timeseries"));
         genericFuncsHeaders.put("# Others", Collections.singletonList("hivemall.tools"));