diff --git a/regression-test/suites/external_table_emr_p2/es/test_external_catalog_es.groovy b/regression-test/suites/external_table_emr_p2/es/test_external_catalog_es.groovy new file mode 100644 index 00000000000000..1234d22d51bad6 --- /dev/null +++ b/regression-test/suites/external_table_emr_p2/es/test_external_catalog_es.groovy @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +//import org.postgresql.Driver +suite("test_external_catalog_es", "p2") { + + String enabled = context.config.otherConfigs.get("enableExternalEsTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extEsHost = context.config.otherConfigs.get("extEsHost") + String extEsPort = context.config.otherConfigs.get("extEsPort") + String extEsUser = context.config.otherConfigs.get("extEsUser") + String extEsPassword = context.config.otherConfigs.get("extEsPassword") + String esCatalogName ="es7_catalog_name" + + String jdbcPg14Database1 = "jdbc_es_14_database1" + String jdbcPg14Table1 = "accounts" + + sql """drop catalog if exists ${esCatalogName}""" + + sql """ + CREATE CATALOG ${esCatalogName} PROPERTIES ( + "type"="es", + "elasticsearch.hosts"="http://${extEsHost}:${extEsPort}", + "elasticsearch.nodes_discovery"="false", + "elasticsearch.username"="${extEsUser}", + "elasticsearch.username"="${extEsPassword}", + ); + """ + + sql """ + SWITCH ${esCatalogName}; + """ + sql """ + SHOW DATABASES; + """ + + def res1=sql "select * from ${jdbcPg14Table1} limit 10;" + logger.info("recoding all: " + res1.toString()) + + sql """drop table if exists ${jdbcPg14Table1};""" + sql """drop database if exists ${jdbcPg14Database1};""" + + sql """switch internal;""" + + } +} diff --git a/regression-test/suites/external_table_emr_p2/es/test_external_es.groovy b/regression-test/suites/external_table_emr_p2/es/test_external_es.groovy index 323a35da3ca148..f5a902a3a7e35f 100644 --- a/regression-test/suites/external_table_emr_p2/es/test_external_es.groovy +++ b/regression-test/suites/external_table_emr_p2/es/test_external_es.groovy @@ -38,11 +38,10 @@ suite("test_external_es", "p2") { `age` varchar(20) COMMENT "" ) ENGINE=ELASTICSEARCH PROPERTIES ( - "hosts" = "https://${extEsHost}:${extEsPort}", + "hosts" = "http://${extEsHost}:${extEsPort}", "index" = "helloworld", "user" = "${extEsUser}", - "password" = "${extEsPassword}", - "http_ssl_enabled" = "true" + "password" = "${extEsPassword}" ); """ def res=sql """show create table ${jdbcPg14Table1};""" diff --git a/regression-test/suites/external_table_emr_p2/mysql/test_external_catalog_mysql.groovy b/regression-test/suites/external_table_emr_p2/mysql/test_external_catalog_mysql.groovy new file mode 100644 index 00000000000000..a411926d99ab41 --- /dev/null +++ b/regression-test/suites/external_table_emr_p2/mysql/test_external_catalog_mysql.groovy @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +//import com.mysql.cj.jdbc.Driver +suite("test_external_catalog_mysql", "p2") { + + String enabled = context.config.otherConfigs.get("enableExternalMysqlTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extMysqlHost = context.config.otherConfigs.get("extMysqlHost") + String extMysqlPort = context.config.otherConfigs.get("extMysqlPort") + String extMysqlUser = context.config.otherConfigs.get("extMysqlUser") + String extMysqlPassword = context.config.otherConfigs.get("extMysqlPassword") + String mysqlDatabaseName01 = "external_mysql_catalog_database01" + String mysqlResource01 = "mysql_catalog_resource_01" + String mysqlCatalogName = "mysql_jdbc" + String mysqlTableName01 = "lineorder" + String mysqlTableName02 = "customer" + String mysqlTableName03 = "supplier" + + sql """drop database if exists ${mysqlDatabaseName01};""" + sql """create database ${mysqlDatabaseName01};""" + sql """use ${mysqlDatabaseName01};""" + + + sql """drop resource if exists ${mysqlResource01};""" + sql """ + CREATE RESOURCE ${mysqlResource01} + properties ( + "type"="jdbc", + "user"="${extMysqlUser}", + "password"="${extMysqlPassword}", + "jdbc_url"="jdbc:mysql://${extMysqlHost}:${extMysqlPort}/ssb?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&serverTimezone=Asia/Shanghai&useSSL=false", + "driver_url"="https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar", + "driver_class"="com.mysql.cj.jdbc.Driver" + ); + """ + + sql """drop catalog if exists ${mysqlCatalogName};""" + + sql """CREATE CATALOG ${mysqlCatalogName} WITH RESOURCE ${mysqlResource01};""" +// sql """drop catalog if exists ${mysqlCatalogName};""" + + +// sql """ +// CREATE CATALOG ${mysqlCatalogName} +// PROPERTIES( +// "type"="jdbc", +// "jdbc.user"="${extMysqlUser}", +// "jdbc.password"="${extMysqlPassword}", +// "jdbc.jdbc_url"="jdbc:mysql://${extMysqlHost}:${extMysqlPort}/ssb?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&serverTimezone=Asia/Shanghai&useSSL=false", +// "jdbc.driver_url"="https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar", +// "jdbc.driver_class"="com.mysql.cj.jdbc.Driver" +// ); +// """ + + sql """switch ${mysqlCatalogName}""" + + sql """use ssb;""" + + + def res = sql """select count(*) from ${mysqlTableName02};""" + logger.info("recoding select: " + res.toString()) + + + def res1 = sql """select count(*) from ${mysqlTableName03};""" + logger.info("recoding select: " + res1.toString()) + + def res2 = sql """select * from ${mysqlTableName02} limit 10;""" + logger.info("recoding select: " + res2.toString()) + + def res3 = sql """select * from ${mysqlTableName03} limit 10;""" + logger.info("recoding select: " + res3.toString()) + + def res4 = sql """select * from ${mysqlTableName02} a join ${mysqlTableName03} b on a.c_nation =b.s_nation limit 5;""" + logger.info("recoding select: " + res4.toString()) + + +// sql """drop table if exists ${mysqlTableName01}""" + sql """drop database if exists ${mysqlDatabaseName01};""" + sql """switch internal;""" + } +} + + + + + + + diff --git a/regression-test/suites/external_table_emr_p2/mysql/test_external_mysql.groovy b/regression-test/suites/external_table_emr_p2/mysql/test_external_mysql.groovy deleted file mode 100644 index 4f366a1c99b20f..00000000000000 --- a/regression-test/suites/external_table_emr_p2/mysql/test_external_mysql.groovy +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -//import com.mysql.cj.jdbc.Driver -suite("test_external_mysql", "p2") { - - String enabled = context.config.otherConfigs.get("enableExternalMysqlTest") - if (enabled != null && enabled.equalsIgnoreCase("true")) { - String extMysqlHost = context.config.otherConfigs.get("extMysqlHost") - String extMysqlPort = context.config.otherConfigs.get("extMysqlPort") - String extMysqlUser = context.config.otherConfigs.get("extMysqlUser") - String extMysqlPassword = context.config.otherConfigs.get("extMysqlPassword") - String mysqlDatabaseName01 = "external_mysql_database01" - String mysqlTableName01 = "external_mysql_table01" - - sql """drop database if exists ${mysqlDatabaseName01};""" - sql """create database ${mysqlDatabaseName01};""" - sql """use ${mysqlDatabaseName01};""" - - sql """drop table if exists ${mysqlTableName01}""" - - sql """ - CREATE EXTERNAL TABLE ${mysqlTableName01} - ( - `id` int, - `name` varchar(128) - ) - ENGINE = mysql - PROPERTIES - ( - "host" = "${extMysqlHost}", - "port" = "${extMysqlPort}", - "user" = "${extMysqlUser}", - "password" = "${extMysqlPassword}", - "database" = "doris_test", - "table" = "ex_tb0", - "charset" = "utf8mb4" - ); - """ - // ERROR 1105 (HY000): errCode = 2, detailMessage = Don't support MySQL table, you should rebuild Doris with WITH_MYSQL option ON - - def res = sql """select count(*) from ${mysqlTableName01}""" - logger.info("recoding select: " + res.toString()) - - - sql """drop table if exists ${mysqlTableName01}""" - sql """drop database if exists ${mysqlDatabaseName01};""" - - } -} - - - - - - - diff --git a/regression-test/suites/external_table_emr_p2/mysql/test_external_resource_mysql.groovy b/regression-test/suites/external_table_emr_p2/mysql/test_external_resource_mysql.groovy index d64a478ad71026..86d432d96680b9 100644 --- a/regression-test/suites/external_table_emr_p2/mysql/test_external_resource_mysql.groovy +++ b/regression-test/suites/external_table_emr_p2/mysql/test_external_resource_mysql.groovy @@ -24,9 +24,17 @@ suite("test_external_resource_mysql", "p2") { String extMysqlUser = context.config.otherConfigs.get("extMysqlUser") String extMysqlPassword = context.config.otherConfigs.get("extMysqlPassword") String mysqlResourceName = "jdbc_resource_mysql_57" - String mysqlDatabaseName01 = "external_mysql_database01" - String mysqlTableName01 = "external_mysql_table01" - String mysqlTableName02 = "external_mysql_table02" + String mysqlDatabaseName01 = "external_mysql_database_ssb" + String mysqlTableNameLineOrder = "external_mysql_table_lineorder" + String mysqlTableNameCustomer = "external_mysql_table_customer" + String mysqlTableNameSupplier = "external_mysql_table_supplier" + + + + sql """drop database if exists ${mysqlDatabaseName01};""" + sql """create database ${mysqlDatabaseName01};""" + sql """use ${mysqlDatabaseName01};""" + sql """drop resource if exists ${mysqlResourceName};""" sql """ @@ -35,40 +43,102 @@ suite("test_external_resource_mysql", "p2") { "type"="jdbc", "user"="${extMysqlUser}", "password"="${extMysqlPassword}", - "jdbc_url"="jdbc:mysql://${extMysqlHost}:${extMysqlPort}/doris_test?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&serverTimezone=Asia/Shanghai&useSSL=false", + "jdbc_url"="jdbc:mysql://${extMysqlHost}:${extMysqlPort}/ssb?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&serverTimezone=Asia/Shanghai&useSSL=false", "driver_url"="https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar", "driver_class"="com.mysql.cj.jdbc.Driver" ); """ - sql """drop database if exists ${mysqlDatabaseName01};""" - sql """create database ${mysqlDatabaseName01};""" - sql """use ${mysqlDatabaseName01};""" - sql """drop table if exists ${mysqlTableName01}""" + + sql """drop table if exists ${mysqlTableNameLineOrder}""" sql """ - CREATE EXTERNAL TABLE ${mysqlTableName01} ( - `id` int, - `name` varchar(128) + CREATE EXTERNAL TABLE ${mysqlTableNameLineOrder} ( + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` bigint(20) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` bigint(20) NOT NULL COMMENT "", + `lo_extendedprice` bigint(20) NOT NULL COMMENT "", + `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", + `lo_discount` bigint(20) NOT NULL COMMENT "", + `lo_revenue` bigint(20) NOT NULL COMMENT "", + `lo_supplycost` bigint(20) NOT NULL COMMENT "", + `lo_tax` bigint(20) NOT NULL COMMENT "", + `lo_commitdate` bigint(20) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" ) ENGINE=JDBC PROPERTIES ( "resource" = "${mysqlResourceName}", - "table" = "ex_tb0", + "table" = "lineorder", "table_type"="mysql" ); """ - def res = sql """select count(*) from ${mysqlTableName01};""" + def res = sql """select * from ${mysqlTableNameLineOrder} limit 10;""" logger.info("recoding select: " + res.toString()) + sql """drop table if exists ${mysqlTableNameCustomer}""" sql """ - CREATE EXTERNAL TABLE ${mysqlTableName02} LIKE ${mysqlTableName01}; + CREATE EXTERNAL TABLE ${mysqlTableNameCustomer} ( + `c_custkey` int(11) DEFAULT NULL, + `c_name` varchar(25) NOT NULL, + `c_address` varchar(40) NOT NULL, + `c_city` varchar(10) NOT NULL, + `c_nation` varchar(15) NOT NULL, + `c_region` varchar(12) NOT NULL, + `c_phone` varchar(15) NOT NULL + ) ENGINE=JDBC + PROPERTIES ( + "resource" = "${mysqlResourceName}", + "table" = "customer", + "table_type"="mysql" + ); """ - res = sql """select count(*) from ${mysqlTableName02};""" - logger.info("recoding select: " + res.toString()) - sql """drop table if exists ${mysqlTableName01}""" - sql """drop table if exists ${mysqlTableName02}""" - sql """drop database if exists ${mysqlDatabaseName01};""" + def res1 = sql """select * from ${mysqlTableNameCustomer} where c_custkey >100 limit 10;""" + logger.info("recoding select: " + res1.toString()) + + def res2 = sql """select * from ${mysqlTableNameCustomer} order by c_custkey desc limit 10;""" + logger.info("recoding select: " + res2.toString()) + +// def res3 = sql """select AVG(lo_discount) from ${mysqlTableNameCustomer} limit 10;""" +// logger.info("recoding select: " + res3.toString()) +// +// def res4 = sql """select MAX(lo_discount) from ${mysqlTableNameCustomer} limit 10;""" +// logger.info("recoding select: " + res4.toString()) + + def res5 = sql """select count(*) from ${mysqlTableNameCustomer};""" + logger.info("recoding select: " + res5.toString()) + + sql """drop table if exists ${mysqlTableNameSupplier}""" + sql """ + CREATE EXTERNAL TABLE ${mysqlTableNameSupplier} ( + `s_suppkey` int(11) DEFAULT NULL, + `s_name` varchar(25) NOT NULL, + `s_address` varchar(25) NOT NULL, + `s_city` varchar(10) NOT NULL, + `s_nation` varchar(15) NOT NULL, + `s_region` varchar(12) NOT NULL, + `s_phone` varchar(15) NOT NULL + ) ENGINE=JDBC + PROPERTIES ( + "resource" = "${mysqlResourceName}", + "table" = "customer", + "table_type"="mysql" + ); + """ + def res6 = sql """select count(*) from ${mysqlTableNameSupplier};""" + logger.info("recoding select: " + res6.toString()) + + def res7 = sql """select * from ${mysqlTableNameCustomer} a join ${mysqlTableNameSupplier} b on a.c_nation =b.s_nation limit 5;""" + logger.info("recoding select: " + res7.toString()) + + sql """drop table if exists ${mysqlTableNameLineOrder}""" + sql """drop database if exists ${mysqlDatabaseName01};""" } } diff --git a/regression-test/suites/external_table_emr_p2/pg/test_external_pg.groovy b/regression-test/suites/external_table_emr_p2/pg/test_external_pg.groovy index 58b2f6b94652d2..e39fb44f76ce15 100644 --- a/regression-test/suites/external_table_emr_p2/pg/test_external_pg.groovy +++ b/regression-test/suites/external_table_emr_p2/pg/test_external_pg.groovy @@ -25,7 +25,10 @@ suite("test_external_pg", "p2") { String extPgPassword = context.config.otherConfigs.get("extPgPassword") String jdbcResourcePg14 = "jdbc_resource_pg_14" String jdbcPg14Database1 = "jdbc_pg_14_database1" - String jdbcPg14Table1 = "jdbc_pg_14_table1" + String pgTableNameLineOrder = "jdbc_pg_14_table1" + String pgTableNameCustomer = "jdbc_pg_14_customer" + String pgTableNameSupplier = "jdbc_pg_14_supplier" + sql """drop database if exists ${jdbcPg14Database1};""" @@ -38,29 +41,95 @@ suite("test_external_pg", "p2") { "type"="jdbc", "user"="${extPgUser}", "password"="${extPgPassword}", - "jdbc_url"="jdbc:postgresql://${extPgHost}:${extPgPort}/exter_test?currentSchema=public", + "jdbc_url"="jdbc:postgresql://${extPgHost}:${extPgPort}/ssb?currentSchema=ssb", "driver_url"="https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/postgresql-42.5.0.jar", "driver_class"="org.postgresql.Driver" ); """ - sql """drop table if exists ${jdbcPg14Table1}""" + sql """drop table if exists ${pgTableNameLineOrder}""" sql """ - CREATE EXTERNAL TABLE ${jdbcPg14Table1} ( - id int, - name char(100) + CREATE EXTERNAL TABLE ${pgTableNameLineOrder} ( + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` bigint(20) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` bigint(20) NOT NULL COMMENT "", + `lo_extendedprice` bigint(20) NOT NULL COMMENT "", + `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", + `lo_discount` bigint(20) NOT NULL COMMENT "", + `lo_revenue` bigint(20) NOT NULL COMMENT "", + `lo_supplycost` bigint(20) NOT NULL COMMENT "", + `lo_tax` bigint(20) NOT NULL COMMENT "", + `lo_commitdate` bigint(20) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" ) ENGINE=JDBC PROPERTIES ( "resource" = "${jdbcResourcePg14}", - "table" = "ext_pg0", + "table" = "lineorder", "table_type"="postgresql" ); """ - def res = sql """SELECT id, name FROM ${jdbcPg14Table1};""" + sql """drop table if exists ${pgTableNameCustomer}""" + sql """ + CREATE EXTERNAL TABLE ${pgTableNameCustomer} ( + `c_custkey` int(11) DEFAULT NULL, + `c_name` varchar(25) NOT NULL, + `c_address` varchar(40) NOT NULL, + `c_city` varchar(10) NOT NULL, + `c_nation` varchar(15) NOT NULL, + `c_region` varchar(12) NOT NULL, + `c_phone` varchar(15) NOT NULL + ) ENGINE=JDBC + PROPERTIES ( + "resource" = "${jdbcResourcePg14}", + "table" = "customer", + "table_type"="mysql" + ); + """ + + sql """drop table if exists ${pgTableNameSupplier}""" + sql """ + CREATE EXTERNAL TABLE ${pgTableNameSupplier} ( + `s_suppkey` int(11) DEFAULT NULL, + `s_name` varchar(25) NOT NULL, + `s_address` varchar(25) NOT NULL, + `s_city` varchar(10) NOT NULL, + `s_nation` varchar(15) NOT NULL, + `s_region` varchar(12) NOT NULL, + `s_phone` varchar(15) NOT NULL + ) ENGINE=JDBC + PROPERTIES ( + "resource" = "${jdbcResourcePg14}", + "table" = "customer", + "table_type"="mysql" + ); + """ + + def res = sql """select count(*) from ${pgTableNameCustomer};""" logger.info("recoding select: " + res.toString()) - sql """drop table if exists ${jdbcPg14Table1}""" + def res1 = sql """select * from ${pgTableNameSupplier} limit 10""" + logger.info("recoding select: " + res1.toString()) + + def res2 = sql """select * from ${pgTableNameSupplier} order by s_suppkey desc limit 10;""" + logger.info("recoding select: " + res2.toString()) + + def res3 = sql """select * from ${pgTableNameSupplier} where s_suppkey>100 limit 10;""" + logger.info("recoding select: " + res3.toString()) + + def res4 = sql """select * from ${pgTableNameCustomer} a join ${pgTableNameSupplier} b on a.c_nation =b.s_nation limit 5;""" + logger.info("recoding select: " + res4.toString()) + + sql """drop table if exists ${pgTableNameLineOrder}""" + sql """drop table if exists ${pgTableNameCustomer}""" + sql """drop table if exists ${pgTableNameSupplier}""" + sql """drop database if exists ${jdbcPg14Database1};""" sql """drop resource if exists ${jdbcResourcePg14};"""