From b4448d26e76b544a403ba0d8c6b4e1647479508c Mon Sep 17 00:00:00 2001 From: nextdreamblue Date: Fri, 24 Mar 2023 11:17:21 +0800 Subject: [PATCH] [fix](multi-catalog) add specified_database_list for hms,iceberg Signed-off-by: nextdreamblue --- docs/en/docs/lakehouse/multi-catalog/hive.md | 5 +++++ .../en/docs/lakehouse/multi-catalog/iceberg.md | 5 +++++ .../zh-CN/docs/lakehouse/multi-catalog/hive.md | 5 +++++ .../docs/lakehouse/multi-catalog/iceberg.md | 5 +++++ .../org/apache/doris/catalog/Resource.java | 1 + .../doris/datasource/ExternalCatalog.java | 18 ++++++++++++++++++ .../doris/datasource/HMSExternalCatalog.java | 4 ++++ .../iceberg/IcebergExternalCatalog.java | 4 ++++ 8 files changed, 47 insertions(+) diff --git a/docs/en/docs/lakehouse/multi-catalog/hive.md b/docs/en/docs/lakehouse/multi-catalog/hive.md index 4f6d6bb582fcba..70cfd2fb36b24b 100644 --- a/docs/en/docs/lakehouse/multi-catalog/hive.md +++ b/docs/en/docs/lakehouse/multi-catalog/hive.md @@ -55,6 +55,11 @@ CREATE CATALOG hive PROPERTIES ( ``` In addition to `type` and `hive.metastore.uris` , which are required, you can specify other parameters regarding the connection. + +> `specified_database_list`: +> +> only synchronize the specified databases, split with ','. Default values is '' will synchronize all databases. db name is case sensitive. +> For example, to specify HDFS HA: diff --git a/docs/en/docs/lakehouse/multi-catalog/iceberg.md b/docs/en/docs/lakehouse/multi-catalog/iceberg.md index 4b870dfea86325..42d3d31aab0903 100644 --- a/docs/en/docs/lakehouse/multi-catalog/iceberg.md +++ b/docs/en/docs/lakehouse/multi-catalog/iceberg.md @@ -53,6 +53,11 @@ CREATE CATALOG iceberg PROPERTIES ( ); ``` +> `specified_database_list`: +> +> only synchronize the specified databases, split with ','. Default values is '' will synchronize all databases. db name is case sensitive. +> + ### Iceberg Native Catalog diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md index f87d8cb95df4b9..d5c34a8f8d267f 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md @@ -53,6 +53,11 @@ CREATE CATALOG hive PROPERTIES ( ``` 除了 `type` 和 `hive.metastore.uris` 两个必须参数外，还可以通过更多参数来传递连接所需要的信息。 + +> `specified_database_list`: +> +> 支持只同步指定的同步多个database，以','分隔。默认为''，同步所有database。db名称是大小写敏感的。 +> 如提供 HDFS HA 信息，示例如下： diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md b/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md index 10e12af67598c3..24e3ca20d32c05 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md @@ -51,6 +51,11 @@ CREATE CATALOG iceberg PROPERTIES ( ); ``` +> `specified_database_list`: +> +> 支持只同步指定的同步多个database，以','分隔。默认为''，同步所有database。db名称是大小写敏感的。 +> + ### 基于Iceberg API创建Catalog diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index cbd942b2acf54e..63d411a7699e98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -46,6 +46,7 @@ public abstract class Resource implements Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(OdbcCatalogResource.class); public static final String REFERENCE_SPLIT = "@"; + public static final String SPECIFIED_DATABASE_LIST = "specified_database_list"; public enum ResourceType { UNKNOWN, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 5bda6d6e2cc63e..1fe81ea61db8de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.external.EsExternalDatabase; import org.apache.doris.catalog.external.ExternalDatabase; import org.apache.doris.catalog.external.ExternalTable; @@ -417,4 +418,21 @@ public void dropDatabase(String dbName) { public void createDatabase(long dbId, String dbName) { throw new NotImplementedException(); } + + public Map getSpecifiedDatabaseMap() { + String specifiedDatabaseList = catalogProperty.getOrDefault(Resource.SPECIFIED_DATABASE_LIST, ""); + Map specifiedDatabaseMap = Maps.newHashMap(); + specifiedDatabaseList = specifiedDatabaseList.trim(); + if (specifiedDatabaseList.isEmpty()) { + return specifiedDatabaseMap; + } + String[] databaseList = specifiedDatabaseList.split(","); + for (int i = 0; i < databaseList.length; i++) { + String dbname = databaseList[i].trim(); + if (!dbname.isEmpty()) { + specifiedDatabaseMap.put(dbname, true); + } + } + return specifiedDatabaseMap; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java index 80443ca8b2bbd2..d532c33395188c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java @@ -112,8 +112,12 @@ protected void init() { initCatalogLog.setCatalogId(id); initCatalogLog.setType(InitCatalogLog.Type.HMS); List allDatabases = client.getAllDatabases(); + Map specifiedDatabaseMap = getSpecifiedDatabaseMap(); // Update the db name to id map. for (String dbName : allDatabases) { + if (!specifiedDatabaseMap.isEmpty() && specifiedDatabaseMap.get(dbName) == null) { + continue; + } long dbId; if (dbNameToId != null && dbNameToId.containsKey(dbName)) { dbId = dbNameToId.get(dbName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 0777cef5188c0e..d7846a90128fc2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -67,7 +67,11 @@ protected void init() { initCatalogLog.setCatalogId(id); initCatalogLog.setType(InitCatalogLog.Type.ICEBERG); List allDatabaseNames = listDatabaseNames(); + Map specifiedDatabaseMap = getSpecifiedDatabaseMap(); for (String dbName : allDatabaseNames) { + if (!specifiedDatabaseMap.isEmpty() && specifiedDatabaseMap.get(dbName) == null) { + continue; + } long dbId; if (dbNameToId != null && dbNameToId.containsKey(dbName)) { dbId = dbNameToId.get(dbName);