From 41d02784b18f6bb25b922d63916978ad43cfa4d2 Mon Sep 17 00:00:00 2001 From: aleph-zero Date: Tue, 3 Nov 2015 15:55:42 -0800 Subject: [PATCH 1/2] DRILL-3791: JDBC Tests Adds tests for the JDBC plugin using MySQL and Derby. --- contrib/storage-jdbc/pom.xml | 207 +++++++++++++++++- .../drill/exec/store/jdbc/TestJdbcPlugin.java | 181 --------------- .../store/jdbc/TestJdbcPluginWithDerbyIT.java | 120 ++++++++++ .../store/jdbc/TestJdbcPluginWithMySQLIT.java | 67 ++++++ .../resources/bootstrap-storage-plugins.json | 21 +- .../src/test/resources/derby-test-data.sql | 50 +++++ .../src/test/resources/mysql-test-data.sql | 53 +++++ 7 files changed, 507 insertions(+), 192 deletions(-) delete mode 100644 contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPlugin.java create mode 100644 contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithDerbyIT.java create mode 100644 contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java create mode 100644 contrib/storage-jdbc/src/test/resources/derby-test-data.sql create mode 100644 contrib/storage-jdbc/src/test/resources/mysql-test-data.sql diff --git a/contrib/storage-jdbc/pom.xml b/contrib/storage-jdbc/pom.xml index b930951a52b..7fb6c84adb6 100755 --- a/contrib/storage-jdbc/pom.xml +++ b/contrib/storage-jdbc/pom.xml @@ -28,9 +28,13 @@ contrib/jdbc-storage-plugin - - + + 5.1.36 + drill_derby_test + drill_mysql_test + + org.apache.drill.exec drill-java-exec @@ -69,8 +73,203 @@ derbynet 10.11.1.1 test - + + + mysql + mysql-connector-java + ${mysql.connector.version} + test + - + + + + src/test/resources + true + + + + + maven-surefire-plugin + + + **/* + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.18.1 + + + + integration-test + verify + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.9.1 + + + reserve-network-port + + reserve-network-port + + process-resources + + + derby.reserved.port + mysql.reserved.port + + + + + + + maven-dependency-plugin + 2.8 + + + + unpack + + + + + com.jcabi + mysql-dist + 5.6.14 + ${mysql.classifier} + zip + false + ${project.build.directory}/mysql-dist + + + + + + + + com.btmatthews.maven.plugins.inmemdb + inmemdb-maven-plugin + 1.4.3 + + inmemdb + 11527 + ${skipTests} + + + + run + + run + + pre-integration-test + + true + derby + ${derby.database.name} + root + root + ${derby.reserved.port} + + + + + + + stop + + stop + + post-integration-test + + + + + com.jcabi + jcabi-mysql-maven-plugin + 0.8.2 + + + mysql-test + + classify + start + stop + + + ${mysql.reserved.port} + ${project.build.directory}/mysql-data + true + ${mysql.database.name} + + + + + + org.codehaus.mojo + sql-maven-plugin + 1.5 + + + mysql + mysql-connector-java + ${mysql.connector.version} + + + + com.mysql.jdbc.Driver + root + root + jdbc:mysql://localhost:${mysql.reserved.port}/${mysql.database.name} + + + + create-tables + pre-integration-test + + execute + + + ascending + + ${basedir}/src/test/resources + + mysql-test-data.sql + + + + + + + + + + + maven-failsafe-plugin + + 1 + + ${derby.reserved.port} + ${mysql.reserved.port} + ${mysql.database.name} + + + **/*IT.java + + + + + + + diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPlugin.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPlugin.java deleted file mode 100644 index 2eb419cdaf6..00000000000 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPlugin.java +++ /dev/null @@ -1,181 +0,0 @@ -package org.apache.drill.exec.store.jdbc; - -import static org.junit.Assert.assertEquals; - -import java.net.InetAddress; -import java.sql.Connection; - -import org.apache.commons.dbcp.BasicDataSource; -import org.apache.derby.drda.NetworkServerControl; -import org.apache.drill.BaseTestQuery; -import org.apache.drill.PlanTestBase; -import org.apache.drill.exec.proto.UserBitShared.QueryType; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -public class TestJdbcPlugin extends PlanTestBase { - - static NetworkServerControl server; - - @BeforeClass - public static void setupDefaultTestCluster() throws Exception { - System.setProperty("derby.drda.startNetworkServer", "true"); - server = new NetworkServerControl(InetAddress.getByName("localhost"), - 20000, - "admin", - "admin"); - java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true); - server.start(consoleWriter); - - BasicDataSource source = new BasicDataSource(); - source.setUrl("jdbc:derby://localhost:20000/memory:testDB;create=true"); - source.setDriverClassName("org.apache.derby.jdbc.ClientDriver"); - - final String insertValues1 = "INSERT INTO person VALUES (1, 'Smith', null, '{number:\"123 Main\"}','mtrx', " - + "'xy', 333.333, 444.444, 555.00, TIME('15:09:02'), DATE('1994-02-23'), TIMESTAMP('1962-09-23 03:23:34.234')," - + " 666.66, 1, -1, false)"; - final String insertValues2 = "INSERT INTO person (PersonId) VALUES (null)"; - try (Connection c = source.getConnection()) { - c.createStatement().execute("CREATE TABLE person\n" + - "(\n" + - "PersonID int,\n" + - "LastName varchar(255),\n" + - "FirstName varchar(255),\n" + - "Address varchar(255),\n" + - "City varchar(255),\n" + - "Code char(2),\n" + - "dbl double,\n" + - "flt float,\n" + - "rel real,\n" + - "tm time,\n" + - "dt date,\n" + - "tms timestamp,\n" + - "num numeric(10,2), \n" + - "sm smallint,\n" + - "bi bigint,\n" + - "bool boolean\n" + - - ")"); - - c.createStatement().execute(insertValues1); - c.createStatement().execute(insertValues2); - c.createStatement().execute(insertValues1); - } - - BaseTestQuery.setupDefaultTestCluster(); - } - - @AfterClass - public static void shutdownDb() throws Exception { - server.shutdown(); - } - - @Test - public void validateResult() throws Exception { - // we'll test data except for date, time and timestamps. Derby mangles these due to improper timezone support. - testBuilder() - .sqlQuery( - "select PERSONID, LASTNAME, FIRSTNAME, ADDRESS, CITY, CODE, DBL, FLT, REL, NUM, SM, BI, BOOL from testdb.PERSON") - .ordered() - .baselineColumns("PERSONID", "LASTNAME", "FIRSTNAME", "ADDRESS", "CITY", "CODE", "DBL", "FLT", "REL", - "NUM", "SM", "BI", "BOOL") - .baselineValues(1, "Smith", null, "{number:\"123 Main\"}", "mtrx", "xy", 333.333, 444.444, 555.00, - 666.66, 1, -1l, false) - .baselineValues(null, null, null, null, null, null, null, null, null, null, null, null, null) - .baselineValues(1, "Smith", null, "{number:\"123 Main\"}", "mtrx", "xy", 333.333, 444.444, 555.00, - 666.66, 1, -1l, false) - .build().run(); - } - - @Test - public void queryDefaultSchema() throws Exception { - testNoResult("select * from testdb.PERSON"); - } - - @Test - public void queryDifferentCase() throws Exception { - testNoResult("select * from testdb.person"); - } - - @Test - public void pushdownJoin() throws Exception { - testNoResult("use testdb"); - String query = "select x.PersonId from (select PersonId from person)x " - + "join (select PersonId from person)y on x.PersonId = y.PersonId "; - testPlanMatchingPatterns(query, new String[] {}, new String[] { "Join" }); - - } - - @Test - public void pushdownJoinAndFilterPushDown() throws Exception { - final String query = "select * from \n" + - "testdb.PERSON e\n" + - "INNER JOIN \n" + - "testdb.PERSON s\n" + - "ON e.FirstName = s.FirstName\n" + - "WHERE e.LastName > 'hello'"; - - testPlanMatchingPatterns(query, new String[] {}, new String[] { "Join", "Filter" }); - } - - @Test - public void pushdownAggregation() throws Exception { - final String query = "select count(*) from \n" + - "testdb.PERSON"; - - testPlanMatchingPatterns(query, new String[] {}, new String[] { "Aggregate" }); - } - - @Test - public void pushdownDoubleJoinAndFilter() throws Exception { - final String query = "select * from \n" + - "testdb.PERSON e\n" + - "INNER JOIN \n" + - "testdb.PERSON s\n" + - "ON e.PersonId = s.PersonId\n" + - "INNER JOIN \n" + - "testdb.PERSON ed\n" + - "ON e.PersonId = ed.PersonId\n" + - "WHERE s.FirstName > 'abc' and ed.FirstName > 'efg'"; - testPlanMatchingPatterns(query, new String[] {}, new String[] { "Join", "Filter" }); - } - - @Test - public void showTablesDefaultSchema() throws Exception { - testNoResult("use testdb"); - assertEquals(1, testRunAndPrint(QueryType.SQL, "show tables like 'PERSON'")); - } - - @Test - public void describe() throws Exception { - testNoResult("use testdb"); - assertEquals(16, testRunAndPrint(QueryType.SQL, "describe PERSON")); - } - - @Test - public void ensureDrillFunctionsAreNotPushedDown() throws Exception { - // This should verify that we're not trying to push CONVERT_FROM into the JDBC storage plugin. If were pushing - // this function down, the SQL query would fail. - testNoResult("select CONVERT_FROM(Address, 'JSON') from testdb.person where PersonId = 1"); - } - - @Test - public void pushdownFilter() throws Exception { - testNoResult("use testdb"); - String query = "select * from person where PersonId = 1"; - testPlanMatchingPatterns(query, new String[] {}, new String[] { "Filter" }); - } -} diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithDerbyIT.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithDerbyIT.java new file mode 100644 index 00000000000..37075f864b8 --- /dev/null +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithDerbyIT.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.jdbc; + +import org.apache.drill.PlanTestBase; +import org.apache.drill.exec.proto.UserBitShared; + +import org.joda.time.DateTime; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * JDBC storage plugin tests against Derby. + */ +public class TestJdbcPluginWithDerbyIT extends PlanTestBase { + + @Test + public void validateResult() throws Exception { + + // Skip date, time, and timestamp types since derby mangles these due to improper timezone support. + testBuilder() + .sqlQuery( + "select PERSON_ID, FIRST_NAME, LAST_NAME, ADDRESS, CITY, STATE, ZIP, JSON, BIGINT_FIELD, SMALLINT_FIELD, " + + "NUMERIC_FIELD, BOOLEAN_FIELD, DOUBLE_FIELD, FLOAT_FIELD, REAL_FIELD, TIME_FIELD, TIMESTAMP_FIELD, " + + "DATE_FIELD from derby.DRILL_DERBY_TEST.PERSON") + .ordered() + .baselineColumns("PERSON_ID", "FIRST_NAME", "LAST_NAME", "ADDRESS", "CITY", "STATE", "ZIP", "JSON", + "BIGINT_FIELD", "SMALLINT_FIELD", "NUMERIC_FIELD", "BOOLEAN_FIELD", "DOUBLE_FIELD", + "FLOAT_FIELD", "REAL_FIELD", "TIME_FIELD", "TIMESTAMP_FIELD", "DATE_FIELD") + .baselineValues(1, "first_name_1", "last_name_1", "1401 John F Kennedy Blvd", "Philadelphia", "PA", 19107, "{ a : 5, b : 6 }", 123456L, 1, 10.01, false, 1.0, 1.1, 111.00, new DateTime(1970, 1, 1, 13, 0, 1), new DateTime(2012, 2, 29, 13, 0, 1), new DateTime(2012, 2, 29, 0, 0, 0)) + .baselineValues(2, "first_name_2", "last_name_2", "One Ferry Building", "San Francisco", "CA", 94111, "{ foo : \"abc\" }", 95949L, 2, 20.02, true, 2.0, 2.1, 222.00, new DateTime(1970, 1, 1, 23, 59, 59), new DateTime(1999, 9, 9, 23, 59, 59), new DateTime(1999, 9, 9, 0, 0, 0)) + .baselineValues(3, "first_name_3", "last_name_3", "176 Bowery", "New York", "NY", 10012, "{ z : [ 1, 2, 3 ] }", 45456L, 3, 30.04, true, 3.0, 3.1, 333.00, new DateTime(1970, 1, 1, 11, 34, 21), new DateTime(2011, 10, 30, 11, 34, 21), new DateTime(2011, 10, 30, 0, 0, 0)) + .baselineValues(4, null, null, "2 15th St NW", "Washington", "DC", 20007, "{ z : { a : 1, b : 2, c : 3 } }", -67L, 4, 40.04, false, 4.0, 4.1, 444.00, new DateTime(1970, 1, 1, 16, 0, 1), new DateTime(2015, 6, 1, 16, 0, 1), new DateTime(2015, 6, 1, 0, 0, 0)) + .baselineValues(5, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null) + .build().run(); + } + + @Test + public void pushdownJoin() throws Exception { + testNoResult("use derby"); + String query = "select x.person_id from (select person_id from DRILL_DERBY_TEST.PERSON) x " + + "join (select person_id from DRILL_DERBY_TEST.PERSON) y on x.person_id = y.person_id "; + testPlanMatchingPatterns(query, new String[]{}, new String[]{"Join"}); + } + + @Test + public void pushdownJoinAndFilterPushDown() throws Exception { + final String query = "select * from \n" + + "derby.DRILL_DERBY_TEST.PERSON e\n" + + "INNER JOIN \n" + + "derby.DRILL_DERBY_TEST.PERSON s\n" + + "ON e.FIRST_NAME = s.FIRST_NAME\n" + + "WHERE e.LAST_NAME > 'hello'"; + + testPlanMatchingPatterns(query, new String[] {}, new String[] { "Join", "Filter" }); + } + + @Test + public void pushdownAggregation() throws Exception { + final String query = "select count(*) from derby.DRILL_DERBY_TEST.PERSON"; + testPlanMatchingPatterns(query, new String[] {}, new String[] { "Aggregate" }); + } + + @Test + public void pushdownDoubleJoinAndFilter() throws Exception { + final String query = "select * from \n" + + "derby.DRILL_DERBY_TEST.PERSON e\n" + + "INNER JOIN \n" + + "derby.DRILL_DERBY_TEST.PERSON s\n" + + "ON e.PERSON_ID = s.PERSON_ID\n" + + "INNER JOIN \n" + + "derby.DRILL_DERBY_TEST.PERSON ed\n" + + "ON e.PERSON_ID = ed.PERSON_ID\n" + + "WHERE s.FIRST_NAME > 'abc' and ed.FIRST_NAME > 'efg'"; + testPlanMatchingPatterns(query, new String[] {}, new String[] { "Join", "Filter" }); + } + + @Test + public void showTablesDefaultSchema() throws Exception { + testNoResult("use derby"); + assertEquals(1, testRunAndPrint(UserBitShared.QueryType.SQL, "show tables like 'PERSON'")); + } + + @Test + public void describe() throws Exception { + testNoResult("use derby"); + assertEquals(18, testRunAndPrint(UserBitShared.QueryType.SQL, "describe PERSON")); + } + + @Test + public void ensureDrillFunctionsAreNotPushedDown() throws Exception { + // This should verify that we're not trying to push CONVERT_FROM into the JDBC storage plugin. If were pushing + // this function down, the SQL query would fail. + testNoResult("select CONVERT_FROM(JSON, 'JSON') from derby.DRILL_DERBY_TEST.PERSON where PERSON_ID = 4"); + } + + @Test + public void pushdownFilter() throws Exception { + testNoResult("use derby"); + String query = "select * from DRILL_DERBY_TEST.PERSON where PERSON_ID = 1"; + testPlanMatchingPatterns(query, new String[]{}, new String[]{"Filter"}); + } +} diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java new file mode 100644 index 00000000000..229f22cd383 --- /dev/null +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.jdbc; + +import org.apache.drill.PlanTestBase; + +import org.joda.time.DateTime; + +import org.junit.Test; + + +/** + * JDBC storage plugin tests against MySQL. + */ +public class TestJdbcPluginWithMySQLIT extends PlanTestBase { + + @Test + public void validateResult() throws Exception { + + testBuilder() + .sqlQuery( + "select person_id, first_name, last_name, address, city, state, zip, json, date_field, datetime_field from mysql.`drill_mysql_test`.person") + .ordered() + .baselineColumns("person_id", "first_name", "last_name", "address", "city", "state", "zip", "json", "date_field", "datetime_field") + .baselineValues(1, "first_name_1", "last_name_1", "1401 John F Kennedy Blvd", "Philadelphia", "PA", 19107, "{ a : 5, b : 6 }", new DateTime(2012, 2, 29, 0, 0, 0), new DateTime(2012, 2, 29, 13, 0, 1)) + .baselineValues(2, "first_name_2", "last_name_2", "One Ferry Building", "San Francisco", "CA", 94111, "{ foo : \"abc\" }", new DateTime(1999, 9, 9, 0, 0, 0), new DateTime(1999, 9, 9, 23, 59, 59)) + .baselineValues(3, "first_name_3", "last_name_3", "176 Bowery", "New York", "NY", 10012, "{ z : [ 1, 2, 3 ] }", new DateTime(2011, 10, 30, 0, 0, 0), new DateTime(2011, 10, 30, 11, 34, 21)) + .baselineValues(4, "first_name_5", "last_name_5", "Chestnut Hill", "Boston", "MA", 12467, "{ [ a, b, c ] }", new DateTime(2015, 6, 1, 0, 0, 0), new DateTime(2015, 9, 22, 15, 46, 10)) + .baselineValues(5, null, null, null, null, null, null, null, null, null) + .build().run(); + } + + @Test + public void pushdownJoin() throws Exception { + String query = "select x.person_id from (select person_id from mysql.`drill_mysql_test`.person) x " + + "join (select person_id from mysql.`drill_mysql_test`.person) y on x.person_id = y.person_id "; + testPlanMatchingPatterns(query, new String[]{}, new String[]{"Join"}); + } + + @Test + public void pushdownJoinAndFilterPushDown() throws Exception { + final String query = "select * from " + + "mysql.`drill_mysql_test`.person e " + + "INNER JOIN " + + "mysql.`drill_mysql_test`.person s " + + "ON e.first_name = s.first_name " + + "WHERE e.last_name > 'hello'"; + + testPlanMatchingPatterns(query, new String[] {}, new String[] { "Join", "Filter" }); + } + +} diff --git a/contrib/storage-jdbc/src/test/resources/bootstrap-storage-plugins.json b/contrib/storage-jdbc/src/test/resources/bootstrap-storage-plugins.json index 200ab93e4eb..add98084744 100755 --- a/contrib/storage-jdbc/src/test/resources/bootstrap-storage-plugins.json +++ b/contrib/storage-jdbc/src/test/resources/bootstrap-storage-plugins.json @@ -1,10 +1,17 @@ { - "storage":{ - testdb : { - type:"jdbc", - enabled: true, - driver:"org.apache.derby.jdbc.ClientDriver", - url:"jdbc:derby://localhost:20000/memory:testDB;" + "storage" : { + derby : { + type : "jdbc", + enabled : true, + driver : "org.apache.derby.jdbc.ClientDriver", + url : "jdbc:derby://localhost:${derby.reserved.port}/memory:${derby.database.name};user=root;password=root" + }, + mysql : { + type : "jdbc", + enabled : true, + driver : "com.mysql.jdbc.Driver", + url : "jdbc:mysql://localhost:${mysql.reserved.port}/${mysql.database.name}?user=root&password=root&useJDBCCompliantTimezoneShift=true" + } } - } } + diff --git a/contrib/storage-jdbc/src/test/resources/derby-test-data.sql b/contrib/storage-jdbc/src/test/resources/derby-test-data.sql new file mode 100644 index 00000000000..6ed8bf0a359 --- /dev/null +++ b/contrib/storage-jdbc/src/test/resources/derby-test-data.sql @@ -0,0 +1,50 @@ + +create SCHEMA drill_derby_test; +set schema drill_derby_test; + +create table person ( + person_id INT NOT NULL PRIMARY KEY, + + first_name VARCHAR(255), + last_name VARCHAR(255), + address VARCHAR(255), + city VARCHAR(255), + state CHAR(2), + zip INT, + + json VARCHAR(255), + + bigint_field BIGINT, + smallint_field SMALLINT, + numeric_field NUMERIC(10, 2), + boolean_field BOOLEAN, + double_field DOUBLE, + float_field FLOAT, + real_field REAL, + + time_field TIME, + timestamp_field TIMESTAMP, + date_field DATE +); + +insert into person (person_id, first_name, last_name, address, city, state, zip, json, bigint_field, smallint_field, numeric_field, + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field) + values (1, 'first_name_1', 'last_name_1', '1401 John F Kennedy Blvd', 'Philadelphia', 'PA', 19107, + '{ a : 5, b : 6 }', 123456, 1, 10.01, false, 1.0, 1.1, 111.00, '13:00:01', '2012-02-29 13:00:01', '2012-02-29'); + +insert into person (person_id, first_name, last_name, address, city, state, zip, json, bigint_field, smallint_field, numeric_field, + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field) + values (2, 'first_name_2', 'last_name_2', 'One Ferry Building', 'San Francisco', 'CA', 94111, + '{ foo : "abc" }', 95949, 2, 20.02, true, 2.0, 2.1, 222.00, '23:59:59', '1999-09-09 23:59:59', '1999-09-09'); + +insert into person (person_id, first_name, last_name, address, city, state, zip, json, bigint_field, smallint_field, numeric_field, + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field) + values (3, 'first_name_3', 'last_name_3', '176 Bowery', 'New York', 'NY', 10012, + '{ z : [ 1, 2, 3 ] }', 45456, 3, 30.04, true, 3.0, 3.1, 333.00, '11:34:21', '2011-10-30 11:34:21', '2011-10-30'); + +insert into person (person_id, first_name, last_name, address, city, state, zip, json, bigint_field, smallint_field, numeric_field, + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field) + values (4, NULL, NULL, '2 15th St NW', 'Washington', 'DC', 20007, + '{ z : { a : 1, b : 2, c : 3 } }', -67, 4, 40.04, false, 4.0, 4.1, 444.00, '16:00:01', '2015-06-01 16:00:01', '2015-06-01'); + +insert into person (person_id) values (5); \ No newline at end of file diff --git a/contrib/storage-jdbc/src/test/resources/mysql-test-data.sql b/contrib/storage-jdbc/src/test/resources/mysql-test-data.sql new file mode 100644 index 00000000000..11b716f124f --- /dev/null +++ b/contrib/storage-jdbc/src/test/resources/mysql-test-data.sql @@ -0,0 +1,53 @@ + +set global time_zone = "+00:00"; + +use drill_mysql_test; + +create table person ( + person_id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + + first_name VARCHAR(255), + last_name VARCHAR(255), + address VARCHAR(255), + city VARCHAR(255), + state CHAR(2), + zip INT, + + json VARCHAR(255), + + bigint_field BIGINT, + smallint_field SMALLINT, + numeric_field NUMERIC(10, 2), + boolean_field BOOLEAN, + double_field DOUBLE, + float_field FLOAT, + real_field REAL, + + time_field TIME, + timestamp_field TIMESTAMP, + date_field DATE, + datetime_field DATETIME +) +engine=InnoDB default charset=latin1; + +insert into person (first_name, last_name, address, city, state, zip, bigint_field, smallint_field, numeric_field, + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field, json) + values ('first_name_1', 'last_name_1', '1401 John F Kennedy Blvd', 'Philadelphia', 'PA', 19107, 123456789, 1, 10.01, + false, 1.0, 1.1, 1.2, '13:00:01', '2012-02-29 13:00:01', '2012-02-29', '2012-02-29 13:00:01', '{ a : 5, b : 6 }'); + +insert into person (first_name, last_name, address, city, state, zip, bigint_field, smallint_field, numeric_field, + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field, json) + values ('first_name_2', 'last_name_2', 'One Ferry Building', 'San Francisco', 'CA', 94111, 95949393, 2, 20.03, + true, 2.0, 2.1, 2.2, '23:59:59', '1999-09-09 23:59:59', '1999-09-09', '1999-09-09 23:59:59', '{ foo : "abc" }'); + +insert into person (first_name, last_name, address, city, state, zip, bigint_field, smallint_field, numeric_field, + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field, json) + values ('first_name_3', 'last_name_3', '176 Bowery', 'New York', 'NY', 10012, 45456767, 3, 30.04, + true, 3.0, 3.1, 3.2, '11:34:21', '2011-10-30 11:34:21', '2011-10-30', '2011-10-30 11:34:21', '{ z : [ 1, 2, 3 ] }'); + +insert into person (first_name, last_name, address, city, state, zip, json, bigint_field, smallint_field, numeric_field, + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field) + values ('first_name_5', 'last_name_5', 'Chestnut Hill', 'Boston', 'MA', 12467, '{ [ a, b, c ] }', 123090, -3, 55.12, false, 5.0, + 5.1, 5.55, '16:00:01', '2015-06-02 10:01:01', '2015-06-01', '2015-09-22 15:46:10'); + +insert into person (person_id) values (5); From 79855b3fb6ca849d38ae151bca13e78564762521 Mon Sep 17 00:00:00 2001 From: aleph-zero Date: Fri, 6 Nov 2015 14:36:26 -0800 Subject: [PATCH 2/2] DRILL-3791: JDBC Tests Fixed a bug in the test framework whereby a byte[] value would be converted into a string, making value comparison impossible. --- contrib/storage-jdbc/pom.xml | 5 ++ .../store/jdbc/TestJdbcPluginWithMySQLIT.java | 77 +++++++++++++++++-- .../src/test/resources/mysql-test-data.sql | 55 +++++++++---- .../org/apache/drill/DrillTestWrapper.java | 5 -- 4 files changed, 113 insertions(+), 29 deletions(-) diff --git a/contrib/storage-jdbc/pom.xml b/contrib/storage-jdbc/pom.xml index 7fb6c84adb6..83789f9331b 100755 --- a/contrib/storage-jdbc/pom.xml +++ b/contrib/storage-jdbc/pom.xml @@ -99,6 +99,7 @@ + org.apache.maven.plugins maven-failsafe-plugin 2.18.1 @@ -112,6 +113,7 @@ + org.codehaus.mojo build-helper-maven-plugin 1.9.1 @@ -142,6 +144,7 @@ + com.jcabi mysql-dist 5.6.14 @@ -156,6 +159,7 @@ + com.btmatthews.maven.plugins.inmemdb inmemdb-maven-plugin 1.4.3 @@ -258,6 +262,7 @@ maven-failsafe-plugin 1 + -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 ${derby.reserved.port} ${mysql.reserved.port} diff --git a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java index 229f22cd383..825f5fd1a08 100644 --- a/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java +++ b/contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java @@ -34,15 +34,76 @@ public void validateResult() throws Exception { testBuilder() .sqlQuery( - "select person_id, first_name, last_name, address, city, state, zip, json, date_field, datetime_field from mysql.`drill_mysql_test`.person") + "select person_id, " + + "first_name, last_name, address, city, state, zip, " + + "bigint_field, smallint_field, numeric_field, " + + "boolean_field, double_field, float_field, real_field, " + + "date_field, datetime_field, " + + "json, text_field, tiny_text_field, medium_text_field, long_text_field, " + + "blob_field, bit_field " + + "from mysql.`drill_mysql_test`.person") .ordered() - .baselineColumns("person_id", "first_name", "last_name", "address", "city", "state", "zip", "json", "date_field", "datetime_field") - .baselineValues(1, "first_name_1", "last_name_1", "1401 John F Kennedy Blvd", "Philadelphia", "PA", 19107, "{ a : 5, b : 6 }", new DateTime(2012, 2, 29, 0, 0, 0), new DateTime(2012, 2, 29, 13, 0, 1)) - .baselineValues(2, "first_name_2", "last_name_2", "One Ferry Building", "San Francisco", "CA", 94111, "{ foo : \"abc\" }", new DateTime(1999, 9, 9, 0, 0, 0), new DateTime(1999, 9, 9, 23, 59, 59)) - .baselineValues(3, "first_name_3", "last_name_3", "176 Bowery", "New York", "NY", 10012, "{ z : [ 1, 2, 3 ] }", new DateTime(2011, 10, 30, 0, 0, 0), new DateTime(2011, 10, 30, 11, 34, 21)) - .baselineValues(4, "first_name_5", "last_name_5", "Chestnut Hill", "Boston", "MA", 12467, "{ [ a, b, c ] }", new DateTime(2015, 6, 1, 0, 0, 0), new DateTime(2015, 9, 22, 15, 46, 10)) - .baselineValues(5, null, null, null, null, null, null, null, null, null) - .build().run(); + .baselineColumns("person_id", + "first_name", "last_name", "address", "city", "state", "zip", + "bigint_field", "smallint_field", "numeric_field", + "boolean_field", + "double_field", "float_field", "real_field", + "date_field", "datetime_field", + "json", "text_field", "tiny_text_field", "medium_text_field", "long_text_field", + "blob_field", "bit_field") + .baselineValues(1, + "first_name_1", "last_name_1", "1401 John F Kennedy Blvd", "Philadelphia", "PA", 19107, + 123456789L, 1, 10.01, + false, + 1.0, 1.1, 1.2, + new DateTime(2012, 2, 29, 0, 0, 0), new DateTime(2012, 2, 29, 13, 0, 1), + "{ a : 5, b : 6 }", + "It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout", + "xxx", + "a medium piece of text", + "a longer piece of text this is going on.....", + "this is a test".getBytes(), + true) + .baselineValues(2, + "first_name_2", "last_name_2", "One Ferry Building", "San Francisco", "CA", 94111, + 45456767L, 3, 30.04, + true, + 3.0, 3.1, 3.2, + new DateTime(2011, 10, 30, 0, 0, 0), new DateTime(2011, 10, 30, 11, 34, 21), + "{ z : [ 1, 2, 3 ] }", + "It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout", + "abc", + "a medium piece of text 2", + "somewhat more text", + "this is a test 2".getBytes(), + false) + .baselineValues(3, + "first_name_3", "last_name_3", "176 Bowery", "New York", "NY", 10012, + 123090L, -3, 55.12, + false, + 5.0, 5.1, 5.55, + new DateTime(2015, 6, 1, 0, 0, 0), new DateTime(2015, 9, 22, 15, 46, 10), + "{ [ a, b, c ] }", + "Neque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit", + "abc", + "a medium piece of text 3", + "somewhat more text", + "this is a test 3".getBytes(), + true) + .baselineValues(5, + null, null, null, null, null, null, + null, null, null, + null, + null, null, null, + null, null, + null, + null, + null, + null, + null, + null, + null) + .build().run(); } @Test diff --git a/contrib/storage-jdbc/src/test/resources/mysql-test-data.sql b/contrib/storage-jdbc/src/test/resources/mysql-test-data.sql index 11b716f124f..4401793715f 100644 --- a/contrib/storage-jdbc/src/test/resources/mysql-test-data.sql +++ b/contrib/storage-jdbc/src/test/resources/mysql-test-data.sql @@ -3,6 +3,13 @@ set global time_zone = "+00:00"; use drill_mysql_test; +create table x ( + a BLOB +); + +insert into x (a) values ('this is a test'); + + create table person ( person_id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, @@ -26,28 +33,44 @@ create table person ( time_field TIME, timestamp_field TIMESTAMP, date_field DATE, - datetime_field DATETIME -) -engine=InnoDB default charset=latin1; + datetime_field DATETIME, + + text_field TEXT, + tiny_text_field TINYTEXT, + medium_text_field MEDIUMTEXT, + long_text_field LONGTEXT, + blob_field BLOB, + bit_field BIT +); insert into person (first_name, last_name, address, city, state, zip, bigint_field, smallint_field, numeric_field, - boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field, json) + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field, json, + text_field, tiny_text_field, medium_text_field, long_text_field, blob_field, bit_field) values ('first_name_1', 'last_name_1', '1401 John F Kennedy Blvd', 'Philadelphia', 'PA', 19107, 123456789, 1, 10.01, - false, 1.0, 1.1, 1.2, '13:00:01', '2012-02-29 13:00:01', '2012-02-29', '2012-02-29 13:00:01', '{ a : 5, b : 6 }'); + false, 1.0, 1.1, 1.2, '13:00:01', '2012-02-29 13:00:01', '2012-02-29', '2012-02-29 13:00:01', '{ a : 5, b : 6 }', + 'It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout', 'xxx', + 'a medium piece of text', 'a longer piece of text this is going on.....', + 'this is a test', + 1); insert into person (first_name, last_name, address, city, state, zip, bigint_field, smallint_field, numeric_field, - boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field, json) - values ('first_name_2', 'last_name_2', 'One Ferry Building', 'San Francisco', 'CA', 94111, 95949393, 2, 20.03, - true, 2.0, 2.1, 2.2, '23:59:59', '1999-09-09 23:59:59', '1999-09-09', '1999-09-09 23:59:59', '{ foo : "abc" }'); + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field, json, + text_field, tiny_text_field, medium_text_field, long_text_field, blob_field, bit_field) + values ('first_name_2', 'last_name_2', 'One Ferry Building', 'San Francisco', 'CA', 94111, 45456767, 3, 30.04, + true, 3.0, 3.1, 3.2, '11:34:21', '2011-10-30 11:34:21', '2011-10-30', '2011-10-30 11:34:21', '{ z : [ 1, 2, 3 ] }', + 'It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout', 'abc', + 'a medium piece of text 2', 'somewhat more text', + 'this is a test 2', + 0); insert into person (first_name, last_name, address, city, state, zip, bigint_field, smallint_field, numeric_field, - boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field, json) - values ('first_name_3', 'last_name_3', '176 Bowery', 'New York', 'NY', 10012, 45456767, 3, 30.04, - true, 3.0, 3.1, 3.2, '11:34:21', '2011-10-30 11:34:21', '2011-10-30', '2011-10-30 11:34:21', '{ z : [ 1, 2, 3 ] }'); - -insert into person (first_name, last_name, address, city, state, zip, json, bigint_field, smallint_field, numeric_field, - boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field) - values ('first_name_5', 'last_name_5', 'Chestnut Hill', 'Boston', 'MA', 12467, '{ [ a, b, c ] }', 123090, -3, 55.12, false, 5.0, - 5.1, 5.55, '16:00:01', '2015-06-02 10:01:01', '2015-06-01', '2015-09-22 15:46:10'); + boolean_field, double_field, float_field, real_field, time_field, timestamp_field, date_field, datetime_field, json, + text_field, tiny_text_field, medium_text_field, long_text_field, blob_field, bit_field) + values ('first_name_3', 'last_name_3', '176 Bowery', 'New York', 'NY', 10012, 123090, -3, 55.12, false, 5.0, + 5.1, 5.55, '16:00:01', '2015-06-02 10:01:01', '2015-06-01', '2015-09-22 15:46:10', '{ [ a, b, c ] }', + 'Neque porro quisquam est qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit', 'abc', + 'a medium piece of text 3', 'somewhat more text', + 'this is a test 3', + 1); insert into person (person_id) values (5); diff --git a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java index 60116e2c466..c5735d8ed57 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java +++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java @@ -253,11 +253,6 @@ private Map addToCombinedVectorResults(List record System.out.println(w.getField()); } } - else if (obj instanceof byte[]) { - // Borrowed from parquet-tools, allows printing of varbinary columns as readable strings - // and also matches the data output by 'parquet-tools cat' - obj= new BinaryNode((byte[]) obj).asText(); - } } combinedVectors.get(field).add(obj); }