Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Select from system tables when table based on table function. #55540

Merged
merged 7 commits into from Nov 9, 2023
Merged

Select from system tables when table based on table function. #55540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
15956a6
Select system tables from table function
MikhailBurdukov Oct 12, 2023
98b5706
disable test in fasttest
MikhailBurdukov Oct 12, 2023
29b52cf
update test
MikhailBurdukov Oct 12, 2023
3d2c8fc
Merge branch 'master' into select_system_tables
MikhailBurdukov Oct 16, 2023
02d6343
Using cached_columns in table_functions
MikhailBurdukov Oct 17, 2023
95df3ff
Review changes
MikhailBurdukov Oct 25, 2023
517ce45
Fix
MikhailBurdukov Nov 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/TableFunctions/TableFunctionMySQL.cpp
View file
Open in desktop
Expand Up @@ -66,7 +66,7 @@ StoragePtr TableFunctionMySQL::executeImpl(
const ASTPtr & /*ast_function*/,
ContextPtr context,
const std::string & table_name,
ColumnsDescription /*cached_columns*/,
ColumnsDescription cached_columns,
bool /*is_insert_query*/) const
{
auto res = std::make_shared<StorageMySQL>(
Expand All @@ -76,7 +76,7 @@ StoragePtr TableFunctionMySQL::executeImpl(
configuration->table,
configuration->replace_query,
configuration->on_duplicate_clause,
ColumnsDescription{},
cached_columns,
ConstraintsDescription{},
String{},
context,
Expand Down
4 changes: 2 additions & 2 deletions src/TableFunctions/TableFunctionPostgreSQL.cpp
View file
Open in desktop
Expand Up @@ -20,13 +20,13 @@ namespace ErrorCodes


StoragePtr TableFunctionPostgreSQL::executeImpl(const ASTPtr & /*ast_function*/,
ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const
ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns, bool /*is_insert_query*/) const
{
auto result = std::make_shared<StoragePostgreSQL>(
StorageID(getDatabaseName(), table_name),
connection_pool,
configuration->table,
ColumnsDescription{},
cached_columns,
ConstraintsDescription{},
String{},
context,
Expand Down
4 changes: 3 additions & 1 deletion src/TableFunctions/TableFunctionS3.cpp
View file
Open in desktop
Expand Up @@ -336,7 +336,7 @@ bool TableFunctionS3::supportsReadingSubsetOfColumns(const ContextPtr & context)
return FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(configuration.format, context);
}

StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const
StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns, bool /*is_insert_query*/) const
{
S3::URI s3_uri (configuration.url);

Expand All @@ -345,6 +345,8 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, Context
columns = parseColumnsListFromString(configuration.structure, context);
else if (!structure_hint.empty())
columns = structure_hint;
else if (!cached_columns.empty())
columns = cached_columns;

StoragePtr storage = std::make_shared<StorageS3>(
configuration,
Expand Down
4 changes: 2 additions & 2 deletions src/TableFunctions/TableFunctionSQLite.cpp
View file
Open in desktop
Expand Up @@ -29,13 +29,13 @@ namespace ErrorCodes


StoragePtr TableFunctionSQLite::executeImpl(const ASTPtr & /*ast_function*/,
ContextPtr context, const String & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const
ContextPtr context, const String & table_name, ColumnsDescription cached_columns, bool /*is_insert_query*/) const
{
auto storage = std::make_shared<StorageSQLite>(StorageID(getDatabaseName(), table_name),
sqlite_db,
database_path,
remote_table_name,
ColumnsDescription{}, ConstraintsDescription{}, context);
cached_columns, ConstraintsDescription{}, context);

storage->startup();
return storage;
Expand Down
12 changes: 12 additions & 0 deletions tests/queries/0_stateless/02888_system_tables_with_inaccsessible_table_function.reference
View file
Open in desktop
@@ -0,0 +1,12 @@
tablefunc01 StorageProxy CREATE TABLE default.tablefunc01 (`x` Int32) AS postgresql(\'127.121.0.1:5432\', \'postgres_db\', \'postgres_table\', \'postgres_user\', \'[HIDDEN]\') [] 1 1
tablefunc02 StorageProxy CREATE TABLE default.tablefunc02 (`x` Int32) AS mysql(\'127.123.0.1:3306\', \'mysql_db\', \'mysql_table\', \'mysql_user\', \'[HIDDEN]\') [] 1 1
tablefunc03 StorageProxy CREATE TABLE default.tablefunc03 (`a` Int32) AS sqlite(\'db_path\', \'table_name\') [] 1 1
tablefunc04 StorageProxy CREATE TABLE default.tablefunc04 (`a` Int32) AS mongodb(\'127.0.0.1:27017\', \'test\', \'my_collection\', \'test_user\', \'[HIDDEN]\', \'a Int\') [] 1 1
tablefunc05 StorageProxy CREATE TABLE default.tablefunc05 (`a` Int32) AS redis(\'127.0.0.1:6379\', \'key\', \'key UInt32\') [] 1 1
tablefunc06 StorageProxy CREATE TABLE default.tablefunc06 (`a` Int32) AS s3(\'http://some_addr:9000/cloud-storage-01/data.tsv\', \'M9O7o0SX5I4udXhWxI12\', \'[HIDDEN]\', \'TSV\') [] 1 1
tablefunc01 StorageProxy CREATE TABLE default.tablefunc01 (`x` Int32) AS postgresql(\'127.121.0.1:5432\', \'postgres_db\', \'postgres_table\', \'postgres_user\', \'[HIDDEN]\') [] 1 1
tablefunc02 StorageProxy CREATE TABLE default.tablefunc02 (`x` Int32) AS mysql(\'127.123.0.1:3306\', \'mysql_db\', \'mysql_table\', \'mysql_user\', \'[HIDDEN]\') [] 1 1
tablefunc03 StorageProxy CREATE TABLE default.tablefunc03 (`a` Int32) AS sqlite(\'db_path\', \'table_name\') [] 1 1
tablefunc04 StorageProxy CREATE TABLE default.tablefunc04 (`a` Int32) AS mongodb(\'127.0.0.1:27017\', \'test\', \'my_collection\', \'test_user\', \'[HIDDEN]\', \'a Int\') [] 1 1
tablefunc05 StorageProxy CREATE TABLE default.tablefunc05 (`a` Int32) AS redis(\'127.0.0.1:6379\', \'key\', \'key UInt32\') [] 1 1
tablefunc06 StorageProxy CREATE TABLE default.tablefunc06 (`a` Int32) AS s3(\'http://some_addr:9000/cloud-storage-01/data.tsv\', \'M9O7o0SX5I4udXhWxI12\', \'[HIDDEN]\', \'TSV\') [] 1 1
43 changes: 43 additions & 0 deletions tests/queries/0_stateless/02888_system_tables_with_inaccsessible_table_function.sql
View file
Open in desktop
@@ -0,0 +1,43 @@
-- Tags: no-fasttest
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why it is not suitable for fasttest?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As I understood table function disabled in the Fasttest. In the first commit CI failed because of:

 executeQuery: Code: 46. DB::Exception: Unknown table function postgresql. (UNKNOWN_FUNCTION)

https://s3.amazonaws.com/clickhouse-test-reports/55540/15956a690dd4d2217e8dde383fc40a308630ba04/fast_test.html


DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};

CREATE DATABASE {CLICKHOUSE_DATABASE:Identifier};


CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc01 (x int) AS postgresql('127.121.0.1:5432', 'postgres_db', 'postgres_table', 'postgres_user', '124444');
CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc02 (x int) AS mysql('127.123.0.1:3306', 'mysql_db', 'mysql_table', 'mysql_user','123123');
CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc03 (a int) AS sqlite('db_path', 'table_name');
CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc04 (a int) AS mongodb('127.0.0.1:27017','test', 'my_collection', 'test_user', 'password', 'a Int');
CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc05 (a int) AS redis('127.0.0.1:6379', 'key', 'key UInt32');
CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc06 (a int) AS s3('http://some_addr:9000/cloud-storage-01/data.tsv', 'M9O7o0SX5I4udXhWxI12', '9ijqzmVN83fzD9XDkEAAAAAAAA', 'TSV');


CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc01_without_schema AS postgresql('127.121.0.1:5432', 'postgres_db', 'postgres_table', 'postgres_user', '124444'); -- { serverError 614 }
CREATE TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc02_without_schema AS mysql('127.123.0.1:3306', 'mysql_db', 'mysql_table', 'mysql_user','123123'); -- {serverError 279 }

SELECT name, engine, engine_full, create_table_query, data_paths, notEmpty([metadata_path]), notEmpty([uuid])
FROM system.tables
WHERE name like '%tablefunc%' and database=currentDatabase()
ORDER BY name;

DETACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc01;
DETACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc02;
DETACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc03;
DETACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc04;
DETACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc05;
DETACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc06;

ATTACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc01;
ATTACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc02;
ATTACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc03;
ATTACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc04;
ATTACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc05;
ATTACH TABLE {CLICKHOUSE_DATABASE:Identifier}.tablefunc06;

SELECT name, engine, engine_full, create_table_query, data_paths, notEmpty([metadata_path]), notEmpty([uuid])
FROM system.tables
WHERE name like '%tablefunc%' and database=currentDatabase()
ORDER BY name;

DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};