From 8ffac97682e952f5d2b54fdc1db80aa28b3b744d Mon Sep 17 00:00:00 2001 From: miaoever Date: Fri, 4 Nov 2016 23:31:00 +0800 Subject: [PATCH 1/3] support specifying an ESCAPE character in LIKE and SIMILAR TO expressions. --- .../table/codegen/calls/BuiltInMethods.scala | 2 ++ .../table/codegen/calls/ScalarFunctions.scala | 12 +++++++ .../flink/api/java/batch/sql/SqlITCase.java | 36 +++++++++++++++++++ .../util/CollectionDataSets.java | 17 ++++++--- 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/BuiltInMethods.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/BuiltInMethods.scala index 6a6898d70c573..265f1688f7c7e 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/BuiltInMethods.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/BuiltInMethods.scala @@ -32,4 +32,6 @@ object BuiltInMethods { val LN = Types.lookupMethod(classOf[Math], "log", classOf[Double]) val ABS = Types.lookupMethod(classOf[SqlFunctions], "abs", classOf[Double]) val ABS_DEC = Types.lookupMethod(classOf[SqlFunctions], "abs", classOf[JBigDecimal]) + val LIKE_WITH_ESCAPE = Types.lookupMethod(classOf[SqlFunctions], "like", classOf[String], classOf[String], classOf[String]) + val SIMILAR_WITH_ESCAPE = Types.lookupMethod(classOf[SqlFunctions], "similar", classOf[String], classOf[String], classOf[String]) } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/ScalarFunctions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/ScalarFunctions.scala index a39829c997659..e7c436aeb6bc6 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/ScalarFunctions.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/ScalarFunctions.scala @@ -97,6 +97,12 @@ object ScalarFunctions { BOOLEAN_TYPE_INFO, BuiltInMethod.LIKE.method) + addSqlFunctionMethod( + LIKE, + Seq(STRING_TYPE_INFO, STRING_TYPE_INFO, STRING_TYPE_INFO), + BOOLEAN_TYPE_INFO, + BuiltInMethods.LIKE_WITH_ESCAPE) + addSqlFunctionNotMethod( NOT_LIKE, Seq(STRING_TYPE_INFO, STRING_TYPE_INFO), @@ -108,6 +114,12 @@ object ScalarFunctions { BOOLEAN_TYPE_INFO, BuiltInMethod.SIMILAR.method) + addSqlFunctionMethod( + SIMILAR_TO, + Seq(STRING_TYPE_INFO, STRING_TYPE_INFO, STRING_TYPE_INFO), + BOOLEAN_TYPE_INFO, + BuiltInMethods.SIMILAR_WITH_ESCAPE) + addSqlFunctionNotMethod( NOT_SIMILAR_TO, Seq(STRING_TYPE_INFO, STRING_TYPE_INFO), diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/sql/SqlITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/sql/SqlITCase.java index 5f5051739ce1b..25b6f71abd7f6 100644 --- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/sql/SqlITCase.java +++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/sql/SqlITCase.java @@ -21,6 +21,7 @@ import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.api.java.table.BatchTableEnvironment; +import org.apache.flink.api.java.tuple.Tuple1; import org.apache.flink.api.java.tuple.Tuple3; import org.apache.flink.api.java.tuple.Tuple5; import org.apache.flink.api.scala.batch.utils.TableProgramsTestBase; @@ -138,4 +139,39 @@ public void testJoin() throws Exception { String expected = "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n"; compareResultAsText(results, expected); } + + + @Test + public void testLikeWithEscapeFromDataSet() throws Exception { + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config()); + + DataSet> ds = CollectionDataSets.getTupleDateSet(env); + tableEnv.registerDataSet("DataSetTable", ds, "x"); + + String sqlQuery = "SELECT x FROM DataSetTable WHERE x LIKE '&%%' ESCAPE '&'"; + Table result = tableEnv.sql(sqlQuery); + + DataSet resultSet = tableEnv.toDataSet(result, Row.class); + List results = resultSet.collect(); + String expected = "%leo*\n" + "%deng*\n" + "%hello*\n";; + compareResultAsText(results, expected); + } + + @Test + public void testSimilarWithEscapeFromDataSet() throws Exception { + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config()); + + DataSet> ds = CollectionDataSets.getTupleDateSet(env); + tableEnv.registerDataSet("DataSetTable", ds, "x"); + + String sqlQuery = "SELECT x FROM DataSetTable WHERE x SIMILAR TO '&%(leo|deng|hello)&*' ESCAPE '&'"; + Table result = tableEnv.sql(sqlQuery); + + DataSet resultSet = tableEnv.toDataSet(result, Row.class); + List results = resultSet.collect(); + String expected = "%leo*\n" + "%deng*\n" + "%hello*\n";; + compareResultAsText(results, expected); + } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java index ba48e121a168d..2a9dad957a009 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java +++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java @@ -33,10 +33,7 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.tuple.Tuple3; -import org.apache.flink.api.java.tuple.Tuple5; -import org.apache.flink.api.java.tuple.Tuple7; +import org.apache.flink.api.java.tuple.*; import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; @@ -54,6 +51,18 @@ */ public class CollectionDataSets { + public static DataSet> getTupleDateSet(ExecutionEnvironment env) { + + List> data = new ArrayList<>(); + data.add(new Tuple1("%leo*")); + data.add(new Tuple1(("%deng*"))); + data.add(new Tuple1("%hello*")); + + Collections.shuffle(data); + + return env.fromCollection(data); + } + public static DataSet> get3TupleDataSet(ExecutionEnvironment env) { List> data = new ArrayList<>(); From ef82b4115e3f865b302061a779efcd1dfc810c37 Mon Sep 17 00:00:00 2001 From: miaoever Date: Sun, 20 Nov 2016 17:27:11 +0800 Subject: [PATCH 2/3] modify the testcases for ESCAPE character in LIKE and SIMILAR TO expressions. --- .../flink/api/java/batch/sql/SqlITCase.java | 36 ------- .../expressions/ScalarFunctionsTest.scala | 96 ++++++++++++++++++- .../util/CollectionDataSets.java | 17 +--- 3 files changed, 98 insertions(+), 51 deletions(-) diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/sql/SqlITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/sql/SqlITCase.java index 25b6f71abd7f6..5f5051739ce1b 100644 --- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/sql/SqlITCase.java +++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/sql/SqlITCase.java @@ -21,7 +21,6 @@ import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.api.java.table.BatchTableEnvironment; -import org.apache.flink.api.java.tuple.Tuple1; import org.apache.flink.api.java.tuple.Tuple3; import org.apache.flink.api.java.tuple.Tuple5; import org.apache.flink.api.scala.batch.utils.TableProgramsTestBase; @@ -139,39 +138,4 @@ public void testJoin() throws Exception { String expected = "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n"; compareResultAsText(results, expected); } - - - @Test - public void testLikeWithEscapeFromDataSet() throws Exception { - ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); - BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config()); - - DataSet> ds = CollectionDataSets.getTupleDateSet(env); - tableEnv.registerDataSet("DataSetTable", ds, "x"); - - String sqlQuery = "SELECT x FROM DataSetTable WHERE x LIKE '&%%' ESCAPE '&'"; - Table result = tableEnv.sql(sqlQuery); - - DataSet resultSet = tableEnv.toDataSet(result, Row.class); - List results = resultSet.collect(); - String expected = "%leo*\n" + "%deng*\n" + "%hello*\n";; - compareResultAsText(results, expected); - } - - @Test - public void testSimilarWithEscapeFromDataSet() throws Exception { - ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); - BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config()); - - DataSet> ds = CollectionDataSets.getTupleDateSet(env); - tableEnv.registerDataSet("DataSetTable", ds, "x"); - - String sqlQuery = "SELECT x FROM DataSetTable WHERE x SIMILAR TO '&%(leo|deng|hello)&*' ESCAPE '&'"; - Table result = tableEnv.sql(sqlQuery); - - DataSet resultSet = tableEnv.toDataSet(result, Row.class); - List results = resultSet.collect(); - String expected = "%leo*\n" + "%deng*\n" + "%hello*\n";; - compareResultAsText(results, expected); - } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/expressions/ScalarFunctionsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/expressions/ScalarFunctionsTest.scala index a6975b9998c3e..a73606f5eab62 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/expressions/ScalarFunctionsTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/expressions/ScalarFunctionsTest.scala @@ -218,6 +218,47 @@ class ScalarFunctionsTest extends ExpressionTestBase { "false") } + @Test + def testLikeWithEscape(): Unit = { + testSqlApi( + "f23 LIKE '&%Th_s%' ESCAPE '&'", + "true") + + testSqlApi( + "f23 LIKE '&%%is a%' ESCAPE '&'", + "true" + ) + + testSqlApi( + "f0 LIKE 'Th_s%' ESCAPE '&'", + "true") + + testSqlApi( + "f0 LIKE '%is a%' ESCAPE '&'", + "true") + } + + @Test + def testNotLikeWithEscape(): Unit = { + testSqlApi( + "f23 NOT LIKE '&%Th_s%' ESCAPE '&'", + "false") + + testSqlApi( + "f23 NOT LIKE '&%%is a%' ESCAPE '&'", + "false") + + testSqlApi( + "f0 NOT LIKE 'Th_s%' ESCAPE '&'", + "false" + ) + + testSqlApi( + "f0 NOT LIKE '%is a%' ESCAPE '&'", + "false" + ) + } + @Test def testSimilar(): Unit = { testAllApis( @@ -248,6 +289,52 @@ class ScalarFunctionsTest extends ExpressionTestBase { "false") } + @Test + def testSimilarWithEscape(): Unit = { + testSqlApi( + "f24 SIMILAR TO '&*&__*' ESCAPE '&'", + "true" + ) + + testSqlApi( + "f0 SIMILAR TO '_*' ESCAPE '&'", + "true" + ) + + testSqlApi( + "f24 SIMILAR TO '&*&_This (is)? a (test)+ Strin_*' ESCAPE '&'", + "true" + ) + + testSqlApi( + "f0 SIMILAR TO 'This (is)? a (test)+ Strin_*' ESCAPE '&'", + "true" + ) + } + + @Test + def testNotSimilarWithEscape(): Unit = { + testSqlApi( + "f24 NOT SIMILAR TO '&*&__*' ESCAPE '&'", + "false" + ) + + testSqlApi( + "f0 NOT SIMILAR TO '_*' ESCAPE '&'", + "false" + ) + + testSqlApi( + "f24 NOT SIMILAR TO '&*&_This (is)? a (test)+ Strin_*' ESCAPE '&'", + "false" + ) + + testSqlApi( + "f0 NOT SIMILAR TO 'This (is)? a (test)+ Strin_*' ESCAPE '&'", + "false" + ) + } + // ---------------------------------------------------------------------------------------------- // Math functions // ---------------------------------------------------------------------------------------------- @@ -1028,7 +1115,7 @@ class ScalarFunctionsTest extends ExpressionTestBase { // ---------------------------------------------------------------------------------------------- def testData = { - val testData = new Row(23) + val testData = new Row(25) testData.setField(0, "This is a test String.") testData.setField(1, true) testData.setField(2, 42.toByte) @@ -1052,6 +1139,8 @@ class ScalarFunctionsTest extends ExpressionTestBase { testData.setField(20, 25) // +2-01 testData.setField(21, null) testData.setField(22, BigDecimal("2").bigDecimal) + testData.setField(23, "%This is a test String.") + testData.setField(24, "*_This is a test String.") testData } @@ -1079,6 +1168,9 @@ class ScalarFunctionsTest extends ExpressionTestBase { Types.INTERVAL_MILLIS, Types.INTERVAL_MONTHS, Types.BOOLEAN, - Types.DECIMAL)).asInstanceOf[TypeInformation[Any]] + Types.DECIMAL, + Types.STRING, + Types.STRING)).asInstanceOf[TypeInformation[Any]] + } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java index 2a9dad957a009..ba48e121a168d 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java +++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java @@ -33,7 +33,10 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; -import org.apache.flink.api.java.tuple.*; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.api.java.tuple.Tuple5; +import org.apache.flink.api.java.tuple.Tuple7; import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; @@ -51,18 +54,6 @@ */ public class CollectionDataSets { - public static DataSet> getTupleDateSet(ExecutionEnvironment env) { - - List> data = new ArrayList<>(); - data.add(new Tuple1("%leo*")); - data.add(new Tuple1(("%deng*"))); - data.add(new Tuple1("%hello*")); - - Collections.shuffle(data); - - return env.fromCollection(data); - } - public static DataSet> get3TupleDataSet(ExecutionEnvironment env) { List> data = new ArrayList<>(); From 1f6cf96820755ac6d76685fee53d0e5c3b1bfbc3 Mon Sep 17 00:00:00 2001 From: miaoever Date: Mon, 21 Nov 2016 10:12:33 +0800 Subject: [PATCH 3/3] Fix file line length exceeds 100 issue. --- .../flink/api/table/codegen/calls/BuiltInMethods.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/BuiltInMethods.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/BuiltInMethods.scala index 265f1688f7c7e..c7c7477be2a83 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/BuiltInMethods.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/BuiltInMethods.scala @@ -32,6 +32,8 @@ object BuiltInMethods { val LN = Types.lookupMethod(classOf[Math], "log", classOf[Double]) val ABS = Types.lookupMethod(classOf[SqlFunctions], "abs", classOf[Double]) val ABS_DEC = Types.lookupMethod(classOf[SqlFunctions], "abs", classOf[JBigDecimal]) - val LIKE_WITH_ESCAPE = Types.lookupMethod(classOf[SqlFunctions], "like", classOf[String], classOf[String], classOf[String]) - val SIMILAR_WITH_ESCAPE = Types.lookupMethod(classOf[SqlFunctions], "similar", classOf[String], classOf[String], classOf[String]) + val LIKE_WITH_ESCAPE = Types.lookupMethod(classOf[SqlFunctions], "like", + classOf[String], classOf[String], classOf[String]) + val SIMILAR_WITH_ESCAPE = Types.lookupMethod(classOf[SqlFunctions], "similar", + classOf[String], classOf[String], classOf[String]) }