From 65630aef14bdddbc1622d050bf8eb51d51f2d34b Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Sat, 28 Feb 2026 11:35:00 +0800 Subject: [PATCH] [feat](catalog) Support include_table_list property and refactor lower_case_database/table_names for external catalog (#60580) Add `include_table_list` catalog property to allow users to specify an explicit list of tables (in "db.tbl" format) to include when listing tables from external catalogs such as HMS. When set, only the specified tables are returned instead of fetching the full table list from the remote metastore, which can significantly reduce metadata overhead for catalogs with a large number of tables. Sometimes calling `getAllTables` from HMS will timeout, can use this to avoid. 2. Introduce `lower_case_database_names` catalog property and perform a unified refactoring together with `lower_case_table_names`: - 0: case-sensitive (default) - 1: database names are stored as lowercase - 2: database names are compared case-insensitively - Fix display behavior when lower_case_database_names or lower_case_table_names = 2: database and table names are now displayed using their original case from the remote metastore, rather than the lowercased form. - Fix case-sensitivity handling in SQL statements: when referencing database or table names in queries, the case-sensitivity behavior now correctly respects the corresponding parameter configuration. - Fix case-sensitivity handling in SHOW statements to properly apply the configured case-sensitivity rules. --- .../java/org/apache/doris/catalog/Env.java | 16 + .../apache/doris/catalog/JdbcResource.java | 16 +- .../org/apache/doris/catalog/Resource.java | 4 - .../apache/doris/datasource/CatalogIf.java | 11 + .../doris/datasource/ExternalCatalog.java | 161 ++++- .../doris/datasource/ExternalDatabase.java | 15 +- .../doris/RemoteDorisExternalCatalog.java | 3 +- .../datasource/es/EsExternalCatalog.java | 3 +- .../datasource/hive/HMSExternalCatalog.java | 3 +- .../iceberg/IcebergExternalCatalog.java | 3 +- .../datasource/jdbc/JdbcExternalCatalog.java | 5 +- .../jdbc/client/JdbcClientConfig.java | 5 +- .../lakesoul/LakeSoulExternalCatalog.java | 2 +- .../maxcompute/MaxComputeExternalCatalog.java | 3 +- .../paimon/PaimonExternalCatalog.java | 3 +- .../datasource/test/TestExternalCatalog.java | 3 +- .../TrinoConnectorExternalCatalog.java | 3 +- .../httpv2/rest/TableQueryPlanAction.java | 2 +- .../org/apache/doris/nereids/CTEContext.java | 13 +- .../doris/nereids/StatementContext.java | 22 +- .../rules/analysis/ExpressionAnalyzer.java | 118 +++- .../trees/plans/commands/BackupCommand.java | 3 +- .../trees/plans/commands/RestoreCommand.java | 3 +- .../plans/commands/ShowTableCommand.java | 3 +- .../trees/plans/commands/UpdateCommand.java | 10 +- .../doris/datasource/ExternalCatalogTest.java | 139 ++++ .../datasource/IncludeTableListTest.java | 424 +++++++++++++ .../jdbc/JdbcExternalCatalogTest.java | 7 +- ...meComparedLowercaseMetaCacheFalseTest.java | 134 ++++ ...ameComparedLowercaseMetaCacheTrueTest.java | 134 ++++ ...NameStoredLowercaseMetaCacheFalseTest.java | 138 ++++ ...eNameStoredLowercaseMetaCacheTrueTest.java | 138 ++++ .../lowercase/ShowAndSelectLowercaseTest.java | 599 ++++++++++++++++++ .../hive/test_hive_case_sensibility.out | 2 + .../test_iceberg_hadoop_case_sensibility.out | 5 + .../test_iceberg_hms_case_sensibility.out | 2 + .../test_iceberg_rest_case_sensibility.out | 5 + .../hive/test_hive_case_sensibility.groovy | 2 +- ...est_iceberg_hadoop_case_sensibility.groovy | 4 +- .../test_iceberg_hms_case_sensibility.groovy | 2 +- .../test_iceberg_rest_case_sensibility.groovy | 4 +- ...th_lower_table_conf_show_and_select.groovy | 5 +- 42 files changed, 2070 insertions(+), 107 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ShowAndSelectLowercaseTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 62b5660311b405..b545b3a1cc35b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -7074,6 +7074,22 @@ public static boolean isTableNamesCaseSensitive() { return GlobalVariable.lowerCaseTableNames == 0; } + public static int getLowerCaseTableNames(String catalogName) { + if (catalogName == null) { + return GlobalVariable.lowerCaseTableNames; + } + CatalogIf catalog = getCurrentEnv().getCatalogMgr().getCatalog(catalogName); + return catalog != null ? catalog.getLowerCaseTableNames() : GlobalVariable.lowerCaseTableNames; + } + + public static int getLowerCaseDatabaseNames(String catalogName) { + if (catalogName == null) { + return 0; // InternalCatalog default: case-sensitive + } + CatalogIf catalog = getCurrentEnv().getCatalogMgr().getCatalog(catalogName); + return catalog != null ? catalog.getLowerCaseDatabaseNames() : 0; + } + private static void getTableMeta(OlapTable olapTable, TGetMetaDBMeta dbMeta) { if (LOG.isDebugEnabled()) { LOG.debug("get table meta. table: {}", olapTable.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index 819e4d21a7da21..f1e312faebeb2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -124,10 +124,10 @@ public class JdbcResource extends Resource { TYPE, CREATE_TIME, ONLY_SPECIFIED_DATABASE, - LOWER_CASE_META_NAMES, - META_NAMES_MAPPING, - INCLUDE_DATABASE_LIST, - EXCLUDE_DATABASE_LIST, + ExternalCatalog.LOWER_CASE_META_NAMES, + ExternalCatalog.META_NAMES_MAPPING, + ExternalCatalog.INCLUDE_DATABASE_LIST, + ExternalCatalog.EXCLUDE_DATABASE_LIST, CONNECTION_POOL_MIN_SIZE, CONNECTION_POOL_MAX_SIZE, CONNECTION_POOL_MAX_LIFE_TIME, @@ -145,10 +145,10 @@ public class JdbcResource extends Resource { static { OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ONLY_SPECIFIED_DATABASE, "false"); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(LOWER_CASE_META_NAMES, "false"); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(META_NAMES_MAPPING, ""); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(INCLUDE_DATABASE_LIST, ""); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(EXCLUDE_DATABASE_LIST, ""); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.LOWER_CASE_META_NAMES, "false"); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.META_NAMES_MAPPING, ""); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.INCLUDE_DATABASE_LIST, ""); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.EXCLUDE_DATABASE_LIST, ""); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MIN_SIZE, "1"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_SIZE, "30"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_LIFE_TIME, "1800000"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index 5db82e3875ae33..de0c5542b3eaab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -47,10 +47,6 @@ public abstract class Resource implements Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(OdbcCatalogResource.class); public static final String REFERENCE_SPLIT = "@"; - public static final String INCLUDE_DATABASE_LIST = "include_database_list"; - public static final String EXCLUDE_DATABASE_LIST = "exclude_database_list"; - public static final String LOWER_CASE_META_NAMES = "lower_case_meta_names"; - public static final String META_NAMES_MAPPING = "meta_names_mapping"; public enum ResourceType { UNKNOWN, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index 7259313495c1fa..cdc029c909a7ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; import org.apache.doris.nereids.trees.plans.commands.info.DropBranchInfo; import org.apache.doris.nereids.trees.plans.commands.info.DropTagInfo; +import org.apache.doris.qe.GlobalVariable; import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; @@ -208,6 +209,16 @@ void truncateTable(String dbName, String tableName, PartitionNamesInfo partition String rawTruncateSql) throws DdlException; + /** 0=case-sensitive, 1=stored lowercase, 2=case-insensitive comparison */ + default int getLowerCaseTableNames() { + return GlobalVariable.lowerCaseTableNames; + } + + /** For InternalCatalog, DB names are always case-sensitive (return 0). */ + default int getLowerCaseDatabaseNames() { + return 0; + } + // Convert from remote database name to local database name, overridden by subclass if necessary default String fromRemoteDatabaseName(String remoteDatabaseName) { return remoteDatabaseName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 534ddfc3a72445..d9dfe2822067bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -24,7 +24,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.MysqlDb; -import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.TableIf; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; @@ -67,6 +66,7 @@ import org.apache.doris.persist.TableBranchOrTagInfo; import org.apache.doris.persist.TruncateTableInfo; import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.qe.GlobalVariable; import org.apache.doris.transaction.TransactionManager; import com.google.common.base.Objects; @@ -113,7 +113,11 @@ public abstract class ExternalCatalog public static final boolean DEFAULT_USE_META_CACHE = true; public static final String FOUND_CONFLICTING = "Found conflicting"; + @Deprecated + // use LOWER_CASE_TABLE_NAMES instead public static final String ONLY_TEST_LOWER_CASE_TABLE_NAMES = "only_test_lower_case_table_names"; + public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; + public static final String LOWER_CASE_DATABASE_NAMES = "lower_case_database_names"; // https://help.aliyun.com/zh/emr/emr-on-ecs/user-guide/use-rootpolicy-to-access-oss-hdfs?spm=a2c4g.11186623.help-menu-search-28066.d_0 public static final String OOS_ROOT_POLICY = "oss.root_policy"; @@ -135,6 +139,13 @@ public abstract class ExternalCatalog public static final String TEST_CONNECTION = "test_connection"; public static final boolean DEFAULT_TEST_CONNECTION = false; + public static final String INCLUDE_DATABASE_LIST = "include_database_list"; + public static final String EXCLUDE_DATABASE_LIST = "exclude_database_list"; + public static final String LOWER_CASE_META_NAMES = "lower_case_meta_names"; + public static final String META_NAMES_MAPPING = "meta_names_mapping"; + // db1.tbl1,db2.tbl2,... + public static final String INCLUDE_TABLE_LIST = "include_table_list"; + // Unique id of this catalog, will be assigned after catalog is loaded. @SerializedName(value = "id") protected long id; @@ -170,6 +181,8 @@ public abstract class ExternalCatalog protected MetaCache> metaCache; protected ExecutionAuthenticator executionAuthenticator; protected ThreadPoolExecutor threadPoolWithPreAuth; + // Map lowercase database names to actual remote database names for case-insensitive lookup + private Map lowerCaseToDatabaseName = Maps.newConcurrentMap(); private volatile Configuration cachedConf = null; private byte[] confLock = new byte[0]; @@ -283,9 +296,29 @@ public void checkWhenCreating() throws DdlException { /** * @param dbName - * @return names of tables in specified database + * @return names of tables in specified database, filtered by include_table_list if configured + */ + public final List listTableNames(SessionContext ctx, String dbName) { + makeSureInitialized(); + Map> includeTableMap = getIncludeTableMap(); + if (includeTableMap.containsKey(dbName) && !includeTableMap.get(dbName).isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("get table list from include map. catalog: {}, db: {}, tables: {}", + name, dbName, includeTableMap.get(dbName)); + } + return includeTableMap.get(dbName); + } + return listTableNamesFromRemote(ctx, dbName); + } + + /** + * Subclasses implement this method to list table names from the remote data source. + * + * @param ctx session context + * @param dbName database name + * @return names of tables in the specified database from the remote source */ - public abstract List listTableNames(SessionContext ctx, String dbName); + protected abstract List listTableNamesFromRemote(SessionContext ctx, String dbName); /** * check if the specified table exist. @@ -468,6 +501,7 @@ private List> getFilteredDatabaseNames() { Map includeDatabaseMap = getIncludeDatabaseMap(); Map excludeDatabaseMap = getExcludeDatabaseMap(); + lowerCaseToDatabaseName.clear(); List> remoteToLocalPairs = Lists.newArrayList(); allDatabases = allDatabases.stream().filter(dbName -> { @@ -485,11 +519,21 @@ private List> getFilteredDatabaseNames() { for (String remoteDbName : allDatabases) { String localDbName = fromRemoteDatabaseName(remoteDbName); + // Populate lowercase mapping for case-insensitive lookups + lowerCaseToDatabaseName.put(remoteDbName.toLowerCase(), remoteDbName); + // Apply lower_case_database_names mode to local name + int dbNameMode = getLowerCaseDatabaseNames(); + if (dbNameMode == 1) { + localDbName = localDbName.toLowerCase(); + } else if (dbNameMode == 2) { + // Mode 2: preserve original remote case for display + localDbName = remoteDbName; + } remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName)); } - // Check for conflicts when lower_case_meta_names = true - if (Boolean.parseBoolean(getLowerCaseMetaNames())) { + // Check for conflicts when lower_case_meta_names = true or lower_case_database_names = 2 + if (Boolean.parseBoolean(getLowerCaseMetaNames()) || getLowerCaseDatabaseNames() == 2) { // Map to track lowercase local names and their corresponding remote names Map> lowerCaseToRemoteNames = Maps.newHashMap(); @@ -540,6 +584,7 @@ public synchronized void resetToUninitialized(boolean invalidCache) { synchronized (this.confLock) { this.cachedConf = null; } + this.lowerCaseToDatabaseName.clear(); onClose(); onRefreshCache(invalidCache); } @@ -660,6 +705,12 @@ public ExternalDatabase getDbNullable(String dbName) { realDbName = InfoSchemaDb.DATABASE_NAME; } else if (realDbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) { realDbName = MysqlDb.DATABASE_NAME; + } else { + // Apply case-insensitive lookup for non-system databases + String localDbName = getLocalDatabaseName(realDbName, false); + if (localDbName != null) { + realDbName = localDbName; + } } // must use full qualified name to generate id. @@ -767,7 +818,14 @@ public Optional> getDbForReplay(String if (!isInitialized()) { return Optional.empty(); } - return metaCache.tryGetMetaObj(dbName); + + // Apply case-insensitive lookup with isReplay=true (no remote calls) + String localDbName = getLocalDatabaseName(dbName, true); + if (localDbName == null) { + localDbName = dbName; // Fallback to original name + } + + return metaCache.tryGetMetaObj(localDbName); } /** @@ -890,6 +948,9 @@ public void gsonPostProcess() throws IOException { if (tableAutoAnalyzePolicy == null) { tableAutoAnalyzePolicy = Maps.newHashMap(); } + if (this.lowerCaseToDatabaseName == null) { + this.lowerCaseToDatabaseName = Maps.newConcurrentMap(); + } } public void addDatabaseForTest(ExternalDatabase db) { @@ -1061,11 +1122,38 @@ public void registerDatabase(long dbId, String dbName) { } protected Map getIncludeDatabaseMap() { - return getSpecifiedDatabaseMap(Resource.INCLUDE_DATABASE_LIST); + return getSpecifiedDatabaseMap(ExternalCatalog.INCLUDE_DATABASE_LIST); } protected Map getExcludeDatabaseMap() { - return getSpecifiedDatabaseMap(Resource.EXCLUDE_DATABASE_LIST); + return getSpecifiedDatabaseMap(ExternalCatalog.EXCLUDE_DATABASE_LIST); + } + + protected Map> getIncludeTableMap() { + Map> includeTableMap = Maps.newHashMap(); + String tableList = catalogProperty.getOrDefault(ExternalCatalog.INCLUDE_TABLE_LIST, ""); + if (Strings.isNullOrEmpty(tableList)) { + return includeTableMap; + } + String[] parts = tableList.split(","); + for (String part : parts) { + String dbTbl = part.trim(); + String[] splits = dbTbl.split("\\."); + if (splits.length != 2) { + LOG.warn("debug invalid include table list: {}, ignore", part); + continue; + } + String db = splits[0]; + String tbl = splits[1]; + List tbls = includeTableMap.get(db); + if (tbls == null) { + includeTableMap.put(db, Lists.newArrayList()); + tbls = includeTableMap.get(db); + } + tbls.add(tbl); + } + LOG.info("debug get include table map: {}", includeTableMap); + return includeTableMap; } private Map getSpecifiedDatabaseMap(String catalogPropertyKey) { @@ -1085,17 +1173,64 @@ private Map getSpecifiedDatabaseMap(String catalogPropertyKey) return specifiedDatabaseMap; } - public String getLowerCaseMetaNames() { - return catalogProperty.getOrDefault(Resource.LOWER_CASE_META_NAMES, "false"); + return catalogProperty.getOrDefault(LOWER_CASE_META_NAMES, "false"); } - public int getOnlyTestLowerCaseTableNames() { - return Integer.parseInt(catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, "0")); + @Override + public int getLowerCaseTableNames() { + return Integer.parseInt(catalogProperty.getOrDefault(LOWER_CASE_TABLE_NAMES, + catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, + String.valueOf(GlobalVariable.lowerCaseTableNames)))); + } + + /** + * Get the lower_case_database_names configuration value. + * Returns the mode for database name case handling: + * - 0: Case-sensitive (default) + * - 1: Database names are stored as lowercase + * - 2: Database name comparison is case-insensitive + */ + @Override + public int getLowerCaseDatabaseNames() { + return Integer.parseInt(catalogProperty.getOrDefault(LOWER_CASE_DATABASE_NAMES, "0")); } public String getMetaNamesMapping() { - return catalogProperty.getOrDefault(Resource.META_NAMES_MAPPING, ""); + return catalogProperty.getOrDefault(ExternalCatalog.META_NAMES_MAPPING, ""); + } + + /** + * Get the local database name based on the lower_case_database_names mode. + * Handles case-insensitive database lookup similar to ExternalDatabase.getLocalTableName(). + */ + @Nullable + private String getLocalDatabaseName(String dbName, boolean isReplay) { + String finalName = dbName; + int mode = getLowerCaseDatabaseNames(); + + if (mode == 1) { + // Mode 1: Store as lowercase + finalName = dbName.toLowerCase(); + } else if (mode == 2) { + // Mode 2: Case-insensitive comparison + finalName = lowerCaseToDatabaseName.get(dbName.toLowerCase()); + if (finalName == null && !isReplay) { + // Refresh database list and try again + try { + getFilteredDatabaseNames(); + finalName = lowerCaseToDatabaseName.get(dbName.toLowerCase()); + } catch (Exception e) { + LOG.warn("Failed to refresh database list for catalog {}", getName(), e); + } + } + if (finalName == null && LOG.isDebugEnabled()) { + LOG.debug("Failed to get database name from: {}.{}, isReplay={}", + getName(), dbName, isReplay); + } + } + + return finalName; } public String bindBrokerName() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 76925a42921bfa..2c20daec0bfe1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -199,6 +199,9 @@ private List> listTableNames() { String localTableName = extCatalog.fromRemoteTableName(remoteName, tableName); if (this.isStoredTableNamesLowerCase()) { localTableName = localTableName.toLowerCase(); + } else if (this.isTableNamesCaseInsensitive()) { + // Mode 2: preserve original remote case for display + localTableName = tableName; } lowerCaseToTableName.put(tableName.toLowerCase(), tableName); return Pair.of(tableName, localTableName); @@ -513,10 +516,6 @@ private String getLocalTableName(String tableName, boolean isReplay) { } } } - if (extCatalog.getLowerCaseMetaNames().equalsIgnoreCase("true") - && (this.isTableNamesCaseInsensitive())) { - finalName = tableName.toLowerCase(); - } if (LOG.isDebugEnabled()) { LOG.debug("get table {} from database: {}.{}, final name is: {}, catalog id: {}", tableName, getCatalog().getName(), getFullName(), finalName, getCatalog().getId()); @@ -595,15 +594,11 @@ public boolean registerTable(TableIf tableIf) { } private boolean isStoredTableNamesLowerCase() { - // Because we have added a test configuration item, - // it needs to be judged together with Env.isStoredTableNamesLowerCase() - return Env.isStoredTableNamesLowerCase() || extCatalog.getOnlyTestLowerCaseTableNames() == 1; + return extCatalog.getLowerCaseTableNames() == 1; } private boolean isTableNamesCaseInsensitive() { - // Because we have added a test configuration item, - // it needs to be judged together with Env.isTableNamesCaseInsensitive() - return Env.isTableNamesCaseInsensitive() || extCatalog.getOnlyTestLowerCaseTableNames() == 2; + return extCatalog.getLowerCaseTableNames() == 2; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java index 76d42b70b4260b..40dec9e5795877 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java @@ -197,8 +197,7 @@ protected List listDatabaseNames() { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return dorisRestClient.getTablesNameList(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java index 8e9c6b08d72151..60371365c50536 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java @@ -134,8 +134,7 @@ protected void initLocalObjectsImpl() { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return esRestClient.listTable(enableIncludeHiddenIndex()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 97436485d2fbc0..fe86390b581f93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -171,8 +171,7 @@ public void onClose() { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 80b780f404d16e..59f5525332e525 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -170,8 +170,7 @@ public boolean tableExist(SessionContext ctx, String dbName, String tblName) { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { // On the Doris side, the result of SHOW TABLES for Iceberg external tables includes both tables and views, // so the combined set of tables and views is used here. List tableNames = metadataOps.listTableNames(dbName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 0d23bb1cb43745..c849d779217e7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -105,7 +105,7 @@ public void checkProperties() throws DdlException { } JdbcResource.checkBooleanProperty(JdbcResource.ONLY_SPECIFIED_DATABASE, getOnlySpecifiedDatabase()); - JdbcResource.checkBooleanProperty(JdbcResource.LOWER_CASE_META_NAMES, getLowerCaseMetaNames()); + JdbcResource.checkBooleanProperty(ExternalCatalog.LOWER_CASE_META_NAMES, getLowerCaseMetaNames()); JdbcResource.checkBooleanProperty(JdbcResource.CONNECTION_POOL_KEEP_ALIVE, String.valueOf(isConnectionPoolKeepAlive())); JdbcResource.checkBooleanProperty(JdbcResource.TEST_CONNECTION, String.valueOf(isTestConnection())); @@ -280,8 +280,7 @@ public String fromRemoteDatabaseName(String remoteDatabaseName) { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return jdbcClient.getTablesNameList(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java index 2d13fbc964b6e3..a35f908ed4b709 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.JdbcResource; import org.apache.doris.datasource.CatalogProperty; +import org.apache.doris.datasource.ExternalCatalog; import com.google.common.collect.Maps; @@ -52,8 +53,8 @@ public class JdbcClientConfig implements Cloneable { public JdbcClientConfig() { this.onlySpecifiedDatabase = JdbcResource.getDefaultPropertyValue(JdbcResource.ONLY_SPECIFIED_DATABASE); - this.isLowerCaseMetaNames = JdbcResource.getDefaultPropertyValue(JdbcResource.LOWER_CASE_META_NAMES); - this.metaNamesMapping = JdbcResource.getDefaultPropertyValue(JdbcResource.META_NAMES_MAPPING); + this.isLowerCaseMetaNames = JdbcResource.getDefaultPropertyValue(ExternalCatalog.LOWER_CASE_META_NAMES); + this.metaNamesMapping = JdbcResource.getDefaultPropertyValue(ExternalCatalog.META_NAMES_MAPPING); this.connectionPoolMinSize = Integer.parseInt( JdbcResource.getDefaultPropertyValue(JdbcResource.CONNECTION_POOL_MIN_SIZE)); this.connectionPoolMaxSize = Integer.parseInt( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java index 0d44e41dce9d1f..465f2c78382212 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java @@ -60,7 +60,7 @@ protected List listDatabaseNames() { } @Override - public List listTableNames(SessionContext ctx, String dbName) { + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { // makeSureInitialized(); // List tifs = lakesoulMetadataManager.getTableInfosByNamespace(dbName); // List tableNames = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java index a0270c2d38d3a0..73eb62f5dc2577 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java @@ -286,8 +286,7 @@ public List listPartitionNames(String dbName, String tbl, long skip, lon } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return mcStructureHelper.listTableNames(getClient(), dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index f15309ea0e96bd..87f606dfcca451 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -89,8 +89,7 @@ public boolean tableExist(SessionContext ctx, String dbName, String tblName) { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return metadataOps.listTableNames(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java index 1fd8e3721f3908..57a483f28c7c58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java @@ -85,8 +85,7 @@ public List mockedSchema(String dbName, String tblName) { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { return mockedTableNames(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java index 3627aff636db77..e2db35707ddb7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java @@ -184,8 +184,7 @@ public boolean tableExist(SessionContext ctx, String dbName, String tblName) { } @Override - public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { QualifiedTablePrefix qualifiedTablePrefix = new QualifiedTablePrefix(trinoCatalogHandle.getCatalogName(), dbName); List tables = trinoListTables(qualifiedTablePrefix); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java index c7b2b9e81b436e..87d1eedaa9541d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java @@ -232,7 +232,7 @@ private void handleQuery(ConnectContext context, String requestDb, String reques String dbName = tableQualifier.get(1); String tableName = tableQualifier.get(2); - if (GlobalVariable.lowerCaseTableNames == 0) { + if (Env.getLowerCaseTableNames(InternalCatalog.INTERNAL_CATALOG_NAME) == 0) { if (!(dbName.equals(requestDb) && tableName.equals(requestTable))) { throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "requested database and table must consistent with sql: request [ " diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java index f2266133b31baf..a2710539147221 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CTEContext.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.GlobalVariable; import com.google.common.collect.ImmutableMap; @@ -56,7 +57,7 @@ public CTEContext(CTEId cteId, @Nullable LogicalSubQueryAlias parsedPlan, if ((parsedPlan == null && previousCteContext != null) || (parsedPlan != null && previousCteContext == null)) { throw new AnalysisException("Only first CteContext can contains null cte plan or previousCteContext"); } - this.name = parsedPlan == null ? null : GlobalVariable.lowerCaseTableNames != 0 + this.name = parsedPlan == null ? null : currentLowerCaseTableNames() != 0 ? parsedPlan.getAlias().toLowerCase(Locale.ROOT) : parsedPlan.getAlias(); this.cteContextMap = previousCteContext == null ? ImmutableMap.of() @@ -68,6 +69,14 @@ public CTEContext(CTEId cteId, @Nullable LogicalSubQueryAlias parsedPlan, this.cteId = cteId; } + private static int currentLowerCaseTableNames() { + ConnectContext ctx = ConnectContext.get(); + if (ctx != null && ctx.getCurrentCatalog() != null) { + return ctx.getCurrentCatalog().getLowerCaseTableNames(); + } + return GlobalVariable.lowerCaseTableNames; + } + public void setAnalyzedPlan(LogicalPlan analyzedPlan) { this.analyzedPlan = analyzedPlan; } @@ -86,7 +95,7 @@ public Optional getAnalyzedCTEPlan(String cteName) { * findCTEContext */ public Optional findCTEContext(String cteName) { - if (GlobalVariable.lowerCaseTableNames != 0) { + if (currentLowerCaseTableNames() != 0) { cteName = cteName.toLowerCase(Locale.ROOT); } if (cteName.equals(name)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 6f4b97b9403f69..b3db083dc2a6a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.Partition; @@ -57,6 +58,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.ShortCircuitQueryContext; @@ -313,10 +315,14 @@ public enum TableFrom { private boolean useGatherForIcebergRewrite = false; private boolean hasNestedColumns; + private final Map lowerCaseTableNamesCache = Maps.newHashMap(); + private final Map lowerCaseDatabaseNamesCache = Maps.newHashMap(); + public StatementContext() { this(ConnectContext.get(), null, 0); } + // For StatementScopeIdGenerator public StatementContext(int initialId) { this(ConnectContext.get(), null, initialId); } @@ -328,7 +334,7 @@ public StatementContext(ConnectContext connectContext, OriginStatement originSta /** * StatementContext */ - public StatementContext(ConnectContext connectContext, OriginStatement originStatement, int initialId) { + private StatementContext(ConnectContext connectContext, OriginStatement originStatement, int initialId) { this.connectContext = connectContext; this.originStatement = originStatement; exprIdGenerator = ExprId.createGenerator(initialId); @@ -1185,4 +1191,18 @@ public boolean hasNestedColumns() { public void setHasNestedColumns(boolean hasNestedColumns) { this.hasNestedColumns = hasNestedColumns; } + + public int getLowerCaseTableNames(String catalogName) { + if (catalogName == null) { + return GlobalVariable.lowerCaseTableNames; + } + return lowerCaseTableNamesCache.computeIfAbsent(catalogName, Env::getLowerCaseTableNames); + } + + public int getLowerCaseDatabaseNames(String catalogName) { + if (catalogName == null) { + return 0; + } + return lowerCaseDatabaseNamesCache.computeIfAbsent(catalogName, Env::getLowerCaseDatabaseNames); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index 1b54d797eea7ab..4ec73e7311ad1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -104,7 +104,6 @@ import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.VariableMgr; import org.apache.doris.qe.VariableVarConverters; @@ -896,12 +895,21 @@ private BoundStar bindQualifiedStar(List qualifierStar, List bound // bound slot is `column` and no qualified case 0: return false; - case 1: // bound slot is `table`.`column` - return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)); - case 2:// bound slot is `db`.`table`.`column` - return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(1)); - case 3:// bound slot is `catalog`.`db`.`table`.`column` - return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(2)); + case 1: { // bound slot is `table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + return sameTableName(qualifierStar.get(0), boundSlotQualifier.get(0), lct); + } + case 2: { // bound slot is `db`.`table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + return sameTableName(qualifierStar.get(0), boundSlotQualifier.get(1), lct); + } + case 3: { // bound slot is `catalog`.`db`.`table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + return sameTableName(qualifierStar.get(0), boundSlotQualifier.get(2), lct); + } default: throw new AnalysisException("Not supported qualifier: " + StringUtils.join(qualifierStar, ".")); @@ -913,12 +921,22 @@ private BoundStar bindQualifiedStar(List qualifierStar, List bound case 0: case 1: // bound slot is `table`.`column` return false; - case 2:// bound slot is `db`.`table`.`column` - return compareDbNameIgnoreClusterName(qualifierStar.get(0), boundSlotQualifier.get(0)) - && qualifierStar.get(1).equalsIgnoreCase(boundSlotQualifier.get(1)); - case 3:// bound slot is `catalog`.`db`.`table`.`column` - return compareDbNameIgnoreClusterName(qualifierStar.get(0), boundSlotQualifier.get(1)) - && qualifierStar.get(1).equalsIgnoreCase(boundSlotQualifier.get(2)); + case 2: { // bound slot is `db`.`table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + int lcdb = resolveLowerCaseDatabaseNames(catName); + return compareDbNameIgnoreClusterName( + qualifierStar.get(0), boundSlotQualifier.get(0), lcdb) + && sameTableName(qualifierStar.get(1), boundSlotQualifier.get(1), lct); + } + case 3: { // bound slot is `catalog`.`db`.`table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + int lcdb = resolveLowerCaseDatabaseNames(catName); + return compareDbNameIgnoreClusterName( + qualifierStar.get(0), boundSlotQualifier.get(1), lcdb) + && sameTableName(qualifierStar.get(1), boundSlotQualifier.get(2), lct); + } default: throw new AnalysisException("Not supported qualifier: " + StringUtils.join(qualifierStar, ".") + ".*"); @@ -931,10 +949,15 @@ private BoundStar bindQualifiedStar(List qualifierStar, List bound case 1: // bound slot is `table`.`column` case 2: // bound slot is `db`.`table`.`column` return false; - case 3:// bound slot is `catalog`.`db`.`table`.`column` + case 3: { // bound slot is `catalog`.`db`.`table`.`column` + String catName = extractCatalogName(boundSlotQualifier); + int lct = resolveLowerCaseTableNames(catName); + int lcdb = resolveLowerCaseDatabaseNames(catName); return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)) - && compareDbNameIgnoreClusterName(qualifierStar.get(1), boundSlotQualifier.get(1)) - && qualifierStar.get(2).equalsIgnoreCase(boundSlotQualifier.get(2)); + && compareDbNameIgnoreClusterName( + qualifierStar.get(1), boundSlotQualifier.get(1), lcdb) + && sameTableName(qualifierStar.get(2), boundSlotQualifier.get(2), lct); + } default: throw new AnalysisException("Not supported qualifier: " + StringUtils.join(qualifierStar, ".") + ".*"); @@ -1113,14 +1136,39 @@ private Optional bindNestedFields(UnboundSlot unboundSlot, Slot slot return Optional.of(new Alias(expression, unboundSlot.getName(), slot.getQualifier())); } - public static boolean sameTableName(String boundSlot, String unboundSlot) { - if (GlobalVariable.lowerCaseTableNames != 1) { + public static boolean sameTableName(String boundSlot, String unboundSlot, int lowerCaseTableNames) { + if (lowerCaseTableNames == 0) { return boundSlot.equals(unboundSlot); } else { return boundSlot.equalsIgnoreCase(unboundSlot); } } + /** Extract catalog name from bound slot qualifier, fallback to current catalog */ + private String extractCatalogName(List boundSlotQualifier) { + if (boundSlotQualifier.size() >= 3) { + return boundSlotQualifier.get(boundSlotQualifier.size() - 3); + } + ConnectContext ctx = ConnectContext.get(); + return (ctx != null) ? ctx.getDefaultCatalog() : null; + } + + private int resolveLowerCaseTableNames(String catalogName) { + CascadesContext cascadesCtx = getCascadesContext(); + if (cascadesCtx != null) { + return cascadesCtx.getStatementContext().getLowerCaseTableNames(catalogName); + } + return Env.getLowerCaseTableNames(catalogName); + } + + private int resolveLowerCaseDatabaseNames(String catalogName) { + CascadesContext cascadesCtx = getCascadesContext(); + if (cascadesCtx != null) { + return cascadesCtx.getStatementContext().getLowerCaseDatabaseNames(catalogName); + } + return Env.getLowerCaseDatabaseNames(catalogName); + } + private boolean shouldBindSlotBy(int namePartSize, Slot boundSlot) { return namePartSize <= boundSlot.getQualifier().size() + 1; } @@ -1147,7 +1195,9 @@ private List bindSingleSlotByTable(String table, String name, Scope scope) } List boundSlotQualifier = boundSlot.getQualifier(); String boundSlotTable = boundSlotQualifier.get(boundSlotQualifier.size() - 1); - if (!sameTableName(boundSlotTable, table)) { + String catalogName = extractCatalogName(boundSlotQualifier); + int lctNames = resolveLowerCaseTableNames(catalogName); + if (!sameTableName(boundSlotTable, table, lctNames)) { continue; } // set sql case as alias @@ -1166,7 +1216,11 @@ private List bindSingleSlotByDb(String db, String table, String name, Scop List boundSlotQualifier = boundSlot.getQualifier(); String boundSlotDb = boundSlotQualifier.get(boundSlotQualifier.size() - 2); String boundSlotTable = boundSlotQualifier.get(boundSlotQualifier.size() - 1); - if (!compareDbNameIgnoreClusterName(boundSlotDb, db) || !sameTableName(boundSlotTable, table)) { + String catalogName = extractCatalogName(boundSlotQualifier); + int lctNames = resolveLowerCaseTableNames(catalogName); + int lcdbNames = resolveLowerCaseDatabaseNames(catalogName); + if (!compareDbNameIgnoreClusterName(boundSlotDb, db, lcdbNames) + || !sameTableName(boundSlotTable, table, lctNames)) { continue; } // set sql case as alias @@ -1186,9 +1240,12 @@ private List bindSingleSlotByCatalog(String catalog, String db, String tab String boundSlotCatalog = boundSlotQualifier.get(boundSlotQualifier.size() - 3); String boundSlotDb = boundSlotQualifier.get(boundSlotQualifier.size() - 2); String boundSlotTable = boundSlotQualifier.get(boundSlotQualifier.size() - 1); + String catalogName = extractCatalogName(boundSlotQualifier); + int lctNames = resolveLowerCaseTableNames(catalogName); + int lcdbNames = resolveLowerCaseDatabaseNames(catalogName); if (!boundSlotCatalog.equalsIgnoreCase(catalog) - || !compareDbNameIgnoreClusterName(boundSlotDb, db) - || !sameTableName(boundSlotTable, table)) { + || !compareDbNameIgnoreClusterName(boundSlotDb, db, lcdbNames) + || !sameTableName(boundSlotTable, table, lctNames)) { continue; } // set sql case as alias @@ -1198,20 +1255,23 @@ private List bindSingleSlotByCatalog(String catalog, String db, String tab } /**compareDbNameIgnoreClusterName.*/ - public static boolean compareDbNameIgnoreClusterName(String name1, String name2) { - if (name1.equalsIgnoreCase(name2)) { + public static boolean compareDbNameIgnoreClusterName(String name1, String name2, + int lowerCaseDatabaseNames) { + boolean ignoreCase = (lowerCaseDatabaseNames != 0); + if (ignoreCase ? name1.equalsIgnoreCase(name2) : name1.equals(name2)) { return true; } - String ignoreClusterName1 = name1; + // Strip cluster namespace prefix (before ':') + String stripped1 = name1; int idx1 = name1.indexOf(":"); if (idx1 > -1) { - ignoreClusterName1 = name1.substring(idx1 + 1); + stripped1 = name1.substring(idx1 + 1); } - String ignoreClusterName2 = name2; + String stripped2 = name2; int idx2 = name2.indexOf(":"); if (idx2 > -1) { - ignoreClusterName2 = name2.substring(idx2 + 1); + stripped2 = name2.substring(idx2 + 1); } - return ignoreClusterName1.equalsIgnoreCase(ignoreClusterName2); + return ignoreCase ? stripped1.equalsIgnoreCase(stripped2) : stripped1.equals(stripped2); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java index 5e9f376a5d56bb..a46f645e6d0bcf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java @@ -31,7 +31,6 @@ import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.StmtExecutor; import com.google.common.base.Joiner; @@ -149,7 +148,7 @@ private void checkTableRefWithoutDatabase() throws AnalysisException { private void updateTableRefInfos() throws AnalysisException { Map tblPartsMap; - if (GlobalVariable.lowerCaseTableNames == 0) { + if (Env.getLowerCaseTableNames(InternalCatalog.INTERNAL_CATALOG_NAME) == 0) { // comparisons case sensitive tblPartsMap = Maps.newTreeMap(); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RestoreCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RestoreCommand.java index 2773c2d6064e83..79c47f94d74193 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RestoreCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RestoreCommand.java @@ -38,7 +38,6 @@ import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.StmtExecutor; import com.google.common.base.Joiner; @@ -184,7 +183,7 @@ private void checkTableRefWithoutDatabase() throws AnalysisException { private void updateTableRefInfos() throws AnalysisException { Map tblPartsMap; - if (GlobalVariable.lowerCaseTableNames == 0) { + if (Env.getLowerCaseTableNames(InternalCatalog.INTERNAL_CATALOG_NAME) == 0) { // comparisons case sensitive tblPartsMap = Maps.newTreeMap(); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java index aff66182ebb138..2e1247906cf9ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableCommand.java @@ -39,7 +39,6 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.ShowResultSet; import org.apache.doris.qe.ShowResultSetMetaData; import org.apache.doris.qe.StmtExecutor; @@ -107,7 +106,7 @@ public void validate(ConnectContext ctx) throws AnalysisException { * isShowTablesCaseSensitive */ public boolean isShowTablesCaseSensitive() { - if (GlobalVariable.lowerCaseTableNames == 0) { + if (Env.getLowerCaseTableNames(catalog) == 0) { return CaseSensibility.TABLE.getCaseSensibility(); } return false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java index 133122ad91cd86..9e40034cef6657 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.StmtType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; @@ -224,9 +225,14 @@ public static void checkAssignmentColumn(ConnectContext ctx, List column throw new AnalysisException("column in assignment list is invalid, " + String.join(".", columnNameParts)); } List tableQualifier = RelationUtil.getQualifierName(ctx, tableNameParts); - if (!ExpressionAnalyzer.sameTableName(tableAlias == null ? tableQualifier.get(2) : tableAlias, tableName) + String catalogName = tableQualifier.get(0); + int lctNames = Env.getLowerCaseTableNames(catalogName); + int lcdbNames = Env.getLowerCaseDatabaseNames(catalogName); + if (!ExpressionAnalyzer.sameTableName( + tableAlias == null ? tableQualifier.get(2) : tableAlias, tableName, lctNames) || (dbName != null - && !ExpressionAnalyzer.compareDbNameIgnoreClusterName(tableQualifier.get(1), dbName))) { + && !ExpressionAnalyzer.compareDbNameIgnoreClusterName( + tableQualifier.get(1), dbName, lcdbNames))) { throw new AnalysisException("column in assignment list is invalid, " + String.join(".", columnNameParts)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java index e75925f5203251..68ffe7221bcb3d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java @@ -215,6 +215,145 @@ public void testExternalCatalogFilteredDatabase() throws Exception { Assertions.assertEquals(MysqlStateType.OK, rootCtx.getState().getStateType()); } + @Test + public void testGetIncludeTableMap() throws Exception { + NereidsParser nereidsParser = new NereidsParser(); + + // Test 1: Empty include_table_list + String createStmt = "create catalog test_include_table_empty properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\"\n" + + ");"; + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + TestExternalCatalog ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_empty"); + Map> includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertTrue(includeTableMap.isEmpty()); + + // Test 2: Single table + createStmt = "create catalog test_include_table_single properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_single"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(1, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertEquals(1, includeTableMap.get("db1").size()); + Assertions.assertEquals("tbl1", includeTableMap.get("db1").get(0)); + + // Test 3: Multiple tables in same database + createStmt = "create catalog test_include_table_same_db properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,db1.tbl2,db1.tbl3\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_same_db"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(1, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertEquals(3, includeTableMap.get("db1").size()); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl1")); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl2")); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl3")); + + // Test 4: Multiple tables in different databases + createStmt = "create catalog test_include_table_diff_db properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,db2.tbl2,db3.tbl3\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_diff_db"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(3, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + Assertions.assertTrue(includeTableMap.containsKey("db3")); + Assertions.assertEquals(1, includeTableMap.get("db1").size()); + Assertions.assertEquals(1, includeTableMap.get("db2").size()); + Assertions.assertEquals(1, includeTableMap.get("db3").size()); + Assertions.assertEquals("tbl1", includeTableMap.get("db1").get(0)); + Assertions.assertEquals("tbl2", includeTableMap.get("db2").get(0)); + Assertions.assertEquals("tbl3", includeTableMap.get("db3").get(0)); + + // Test 5: Invalid format (should be ignored) + createStmt = "create catalog test_include_table_invalid properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,invalid_format,db2.tbl2,too.many.dots\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_invalid"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(2, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + Assertions.assertFalse(includeTableMap.containsKey("invalid_format")); + Assertions.assertFalse(includeTableMap.containsKey("too")); + + // Test 6: With whitespace (should be trimmed) + createStmt = "create catalog test_include_table_whitespace properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \" db1.tbl1 , db2.tbl2 \"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_whitespace"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(2, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + + // Test 7: Mixed valid and invalid with multiple tables in same db + createStmt = "create catalog test_include_table_mixed properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,db1.tbl2,invalid,db2.tbl3\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_mixed"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(2, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + Assertions.assertEquals(2, includeTableMap.get("db1").size()); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl1")); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl2")); + Assertions.assertEquals(1, includeTableMap.get("db2").size()); + Assertions.assertTrue(includeTableMap.get("db2").contains("tbl3")); + } + public static class RefreshCatalogProvider implements TestExternalCatalog.TestCatalogProvider { public static final Map>> MOCKED_META; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java new file mode 100644 index 00000000000000..2f8e5d5177c00a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java @@ -0,0 +1,424 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class IncludeTableListTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + private void createCatalog(String catalogName, String providerClass, String includeTableList) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("create catalog ").append(catalogName).append(" properties(\n"); + sb.append(" \"type\" = \"test\",\n"); + sb.append(" \"catalog_provider.class\" = \"").append(providerClass).append("\""); + if (includeTableList != null) { + sb.append(",\n \"").append(ExternalCatalog.INCLUDE_TABLE_LIST) + .append("\" = \"").append(includeTableList).append("\""); + } + sb.append("\n);"); + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(sb.toString()); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + private void dropCatalog(String catalogName) throws Exception { + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog " + catalogName); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + private void refreshCatalog(String catalogName) { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand(catalogName, null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + } + + private static final String PROVIDER_CLASS = + "org.apache.doris.datasource.IncludeTableListTest$IncludeTableListProvider"; + + // ==================== Basic filtering tests ==================== + + /** + * When include_table_list is not configured, all tables should be visible. + */ + @Test + public void testNoIncludeTableList() throws Exception { + String catalogName = "test_no_include"; + createCatalog(catalogName, PROVIDER_CLASS, null); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set tableNames = db1.getTableNamesWithLock(); + // All 3 tables in db1 should be visible + Assertions.assertEquals(3, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertTrue(tableNames.contains("tbl2")); + Assertions.assertTrue(tableNames.contains("tbl3")); + + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set db2TableNames = db2.getTableNamesWithLock(); + Assertions.assertEquals(2, db2TableNames.size()); + Assertions.assertTrue(db2TableNames.contains("tbl_a")); + Assertions.assertTrue(db2TableNames.contains("tbl_b")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list specifies a single table in one db, only that table should be visible + * in that db; other dbs should show all tables. + */ + @Test + public void testSingleTableInclude() throws Exception { + String catalogName = "test_single_include"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1"); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set tableNames = db1.getTableNamesWithLock(); + // Only tbl1 should be visible in db1 + Assertions.assertEquals(1, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertFalse(tableNames.contains("tbl2")); + Assertions.assertFalse(tableNames.contains("tbl3")); + + // db2 should still show all tables (not in include_table_list) + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set db2TableNames = db2.getTableNamesWithLock(); + Assertions.assertEquals(2, db2TableNames.size()); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list specifies multiple tables in same db. + */ + @Test + public void testMultipleTablesInSameDb() throws Exception { + String catalogName = "test_multi_same_db"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1,db1.tbl3"); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set tableNames = db1.getTableNamesWithLock(); + // Only tbl1 and tbl3 should be visible + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertFalse(tableNames.contains("tbl2")); + Assertions.assertTrue(tableNames.contains("tbl3")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list specifies tables across multiple dbs. + */ + @Test + public void testMultipleTablesAcrossDbs() throws Exception { + String catalogName = "test_multi_cross_db"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl2,db2.tbl_a"); + try { + refreshCatalog(catalogName); + + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set db1Tables = db1.getTableNamesWithLock(); + Assertions.assertEquals(1, db1Tables.size()); + Assertions.assertTrue(db1Tables.contains("tbl2")); + + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set db2Tables = db2.getTableNamesWithLock(); + Assertions.assertEquals(1, db2Tables.size()); + Assertions.assertTrue(db2Tables.contains("tbl_a")); + } finally { + dropCatalog(catalogName); + } + } + + // ==================== Error / edge case tests ==================== + + /** + * When include_table_list specifies a table that does NOT exist in the remote source, + * the table name should still appear in listTableNames but getTableNullable should return null. + */ + @Test + public void testNonExistentTableInIncludeList() throws Exception { + String catalogName = "test_nonexist_tbl"; + // "nonexistent_table" does not exist in the provider's db1 + createCatalog(catalogName, PROVIDER_CLASS, "db1.nonexistent_table"); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + // The include list overrides remote listing, so it reports "nonexistent_table" + Set tableNames = db1.getTableNamesWithLock(); + Assertions.assertEquals(1, tableNames.size()); + Assertions.assertTrue(tableNames.contains("nonexistent_table")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * Mix of existing and non-existing tables in include_table_list. + * Existing table should be accessible; non-existing table should return null. + */ + @Test + public void testMixExistentAndNonExistentTables() throws Exception { + String catalogName = "test_mix_exist"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1,db1.no_such_table"); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set tableNames = db1.getTableNamesWithLock(); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertTrue(tableNames.contains("no_such_table")); + + // Existing table should be accessible + Assertions.assertNotNull(db1.getTableNullable("tbl1")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list refers to a non-existent database, that db entry + * in the include map is simply ignored (the db won't appear). + */ + @Test + public void testNonExistentDbInIncludeList() throws Exception { + String catalogName = "test_nonexist_db"; + // "no_such_db" does not exist in the provider + createCatalog(catalogName, PROVIDER_CLASS, "no_such_db.tbl1,db1.tbl1"); + try { + refreshCatalog(catalogName); + // db1 should still work with filtered table + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set db1Tables = db1.getTableNamesWithLock(); + Assertions.assertEquals(1, db1Tables.size()); + Assertions.assertTrue(db1Tables.contains("tbl1")); + + // The non-existent db should return null + Assertions.assertNull( + env.getCatalogMgr().getCatalog(catalogName).getDbNullable("no_such_db")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list contains entries with invalid format (no dot separator), + * those entries are silently ignored. + */ + @Test + public void testInvalidFormatInIncludeList() throws Exception { + String catalogName = "test_invalid_fmt"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1,bad_format,db2.tbl_a,too.many.dots"); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set db1Tables = db1.getTableNamesWithLock(); + // Only "db1.tbl1" is a valid entry for db1 + Assertions.assertEquals(1, db1Tables.size()); + Assertions.assertTrue(db1Tables.contains("tbl1")); + + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set db2Tables = db2.getTableNamesWithLock(); + Assertions.assertEquals(1, db2Tables.size()); + Assertions.assertTrue(db2Tables.contains("tbl_a")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list entries have extra whitespace, they should be trimmed properly. + */ + @Test + public void testWhitespaceInIncludeList() throws Exception { + String catalogName = "test_whitespace"; + createCatalog(catalogName, PROVIDER_CLASS, " db1.tbl1 , db1.tbl2 "); + try { + refreshCatalog(catalogName); + ExternalDatabase db1 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set tableNames = db1.getTableNamesWithLock(); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertTrue(tableNames.contains("tbl2")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list is set but all entries are for one db, + * another db should show all its tables (unaffected). + */ + @Test + public void testUnaffectedDbShowsAllTables() throws Exception { + String catalogName = "test_unaffected_db"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1"); + try { + refreshCatalog(catalogName); + // db2 is not mentioned in include_table_list, so all tables should be visible + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set db2Tables = db2.getTableNamesWithLock(); + Assertions.assertEquals(2, db2Tables.size()); + Assertions.assertTrue(db2Tables.contains("tbl_a")); + Assertions.assertTrue(db2Tables.contains("tbl_b")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * Test that listTableNames (the catalog-level API) returns the included list directly + * when include_table_list is configured for that db. + */ + @Test + public void testListTableNamesAPI() throws Exception { + String catalogName = "test_api_list"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl2,db1.tbl3"); + try { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog(catalogName); + List tableNames = catalog.listTableNames(null, "db1"); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl2")); + Assertions.assertTrue(tableNames.contains("tbl3")); + Assertions.assertFalse(tableNames.contains("tbl1")); + + // db2 not in include map → returns all remote tables + List db2Names = catalog.listTableNames(null, "db2"); + Assertions.assertEquals(2, db2Names.size()); + Assertions.assertTrue(db2Names.contains("tbl_a")); + Assertions.assertTrue(db2Names.contains("tbl_b")); + } finally { + dropCatalog(catalogName); + } + } + + // ==================== Mock data provider ==================== + + public static class IncludeTableListProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + + // db1 with 3 tables + Map> db1Tables = Maps.newHashMap(); + db1Tables.put("tbl1", Lists.newArrayList( + new Column("id", PrimitiveType.INT), + new Column("name", PrimitiveType.VARCHAR))); + db1Tables.put("tbl2", Lists.newArrayList( + new Column("id", PrimitiveType.INT), + new Column("value", PrimitiveType.BIGINT))); + db1Tables.put("tbl3", Lists.newArrayList( + new Column("key", PrimitiveType.VARCHAR), + new Column("data", PrimitiveType.STRING))); + MOCKED_META.put("db1", db1Tables); + + // db2 with 2 tables + Map> db2Tables = Maps.newHashMap(); + db2Tables.put("tbl_a", Lists.newArrayList( + new Column("col1", PrimitiveType.INT), + new Column("col2", PrimitiveType.FLOAT))); + db2Tables.put("tbl_b", Lists.newArrayList( + new Column("x", PrimitiveType.BIGINT), + new Column("y", PrimitiveType.DOUBLE))); + MOCKED_META.put("db2", db2Tables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java index 5a7379b2e842ec..c63aebfcb35b0a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java @@ -21,6 +21,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogFactory; +import org.apache.doris.datasource.ExternalCatalog; import com.google.common.collect.Maps; import org.junit.Assert; @@ -96,14 +97,14 @@ public void checkPropertiesTest() { exception1.getMessage()); jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.ONLY_SPECIFIED_DATABASE, "true"); - jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.LOWER_CASE_META_NAMES, "1"); + jdbcExternalCatalog.getCatalogProperty().addProperty(ExternalCatalog.LOWER_CASE_META_NAMES, "1"); Exception exception2 = Assert.assertThrows(DdlException.class, () -> jdbcExternalCatalog.checkProperties()); Assert.assertEquals("errCode = 2, detailMessage = lower_case_meta_names must be true or false", exception2.getMessage()); jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.ONLY_SPECIFIED_DATABASE, "false"); - jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.LOWER_CASE_META_NAMES, "false"); - jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.INCLUDE_DATABASE_LIST, "db1,db2"); + jdbcExternalCatalog.getCatalogProperty().addProperty(ExternalCatalog.LOWER_CASE_META_NAMES, "false"); + jdbcExternalCatalog.getCatalogProperty().addProperty(ExternalCatalog.INCLUDE_DATABASE_LIST, "db1,db2"); DdlException exceptione3 = Assert.assertThrows(DdlException.class, () -> jdbcExternalCatalog.checkProperties()); Assert.assertEquals( "errCode = 2, detailMessage = include_database_list and exclude_database_list cannot be set when only_specified_database is false", diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java new file mode 100644 index 00000000000000..33eae7ff31251d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 2 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest$ExternalDatabaseNameComparedLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"2\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(2, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with lowercase, should retrieve original case + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("database1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("DATABASE1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DataBase2"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("DATABASE2", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("DATABASE1")); // Original case preserved + Assertions.assertTrue(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameComparedLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with uppercase names that preserve case + MOCKED_META.put("DATABASE1", tables); + MOCKED_META.put("DATABASE2", tables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java new file mode 100644 index 00000000000000..299bdb48cc653a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 2 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest$ExternalDatabaseNameComparedLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"2\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(2, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with lowercase, should retrieve original case + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("database1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("DATABASE1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DataBase2"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("DATABASE2", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("DATABASE1")); // Original case preserved + Assertions.assertTrue(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameComparedLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with uppercase names that preserve case + MOCKED_META.put("DATABASE1", tables); + MOCKED_META.put("DATABASE2", tables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java new file mode 100644 index 00000000000000..626f2494035dd4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 1 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest$ExternalDatabaseNameStoredLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"1\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(1, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with uppercase, should retrieve lowercase + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DATABASE1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("database1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("Database1"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("database1", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("database1")); + Assertions.assertTrue(dbNames.contains("database2")); + Assertions.assertTrue(dbNames.contains("database3")); + Assertions.assertFalse(dbNames.contains("Database1")); + Assertions.assertFalse(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameStoredLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with mixed case in remote system + MOCKED_META.put("Database1", tables); + MOCKED_META.put("DATABASE2", tables); + MOCKED_META.put("database3", tables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java new file mode 100644 index 00000000000000..83268e5704ec5d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 1 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest$ExternalDatabaseNameStoredLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"1\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(1, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with uppercase, should retrieve lowercase + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DATABASE1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("database1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("Database1"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("database1", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("database1")); + Assertions.assertTrue(dbNames.contains("database2")); + Assertions.assertTrue(dbNames.contains("database3")); + Assertions.assertFalse(dbNames.contains("Database1")); + Assertions.assertFalse(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameStoredLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with mixed case in remote system + MOCKED_META.put("Database1", tables); + MOCKED_META.put("DATABASE2", tables); + MOCKED_META.put("database3", tables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ShowAndSelectLowercaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ShowAndSelectLowercaseTest.java new file mode 100644 index 00000000000000..6f7dc84a544421 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ShowAndSelectLowercaseTest.java @@ -0,0 +1,599 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowDatabasesCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowTableCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * SQL-level tests for SHOW TABLES/DATABASES and SELECT query resolution + * under different per-catalog lower_case_table_names / lower_case_database_names modes (0/1/2). + * + * Mode 0: case-sensitive (exact match required) + * Mode 1: stored as lowercase (remote already lowercase; any-case input lowered for lookup) + * Mode 2: case-insensitive comparison (original case preserved; any-case input matches) + */ +public class ShowAndSelectLowercaseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void beforeCluster() { + Config.lower_case_table_names = 0; + FeConstants.runningUnitTest = true; + } + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // Mode 0: mixed-case remote names, case-sensitive + createCatalog("test_mode0", + "org.apache.doris.datasource.lowercase.ShowAndSelectLowercaseTest$MixedCaseProvider", + 0, 0); + // Mode 1: lowercase remote names (real-world: remote stores lowercase) + createCatalog("test_mode1", + "org.apache.doris.datasource.lowercase.ShowAndSelectLowercaseTest$LowercaseProvider", + 1, 1); + // Mode 2: mixed-case remote names, case-insensitive comparison + createCatalog("test_mode2", + "org.apache.doris.datasource.lowercase.ShowAndSelectLowercaseTest$MixedCaseProvider", + 2, 2); + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + dropCatalog("test_mode0"); + dropCatalog("test_mode1"); + dropCatalog("test_mode2"); + } + + // ==================== Helper methods ==================== + + private void createCatalog(String name, String providerClass, + int lowerCaseTableNames, int lowerCaseDatabaseNames) throws Exception { + String createStmt = "create catalog " + name + " properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" = \"" + providerClass + "\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_TABLE_NAMES + "\" = \"" + lowerCaseTableNames + "\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"" + lowerCaseDatabaseNames + "\"\n" + + ");"; + NereidsParser parser = new NereidsParser(); + LogicalPlan plan = parser.parseSingle(createStmt); + if (plan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) plan).run(rootCtx, null); + } + } + + private void dropCatalog(String name) throws Exception { + NereidsParser parser = new NereidsParser(); + LogicalPlan plan = parser.parseSingle("drop catalog " + name); + if (plan instanceof DropCatalogCommand) { + ((DropCatalogCommand) plan).run(rootCtx, null); + } + } + + private List getShowDatabaseNames(String catalog) throws Exception { + rootCtx.setThreadLocalInfo(); + ShowDatabasesCommand cmd = new ShowDatabasesCommand(catalog, null, null); + ShowResultSet resultSet = cmd.doRun(rootCtx, new StmtExecutor(rootCtx, "")); + return resultSet.getResultRows().stream() + .map(row -> row.get(0)) + .filter(n -> !n.equalsIgnoreCase("information_schema") && !n.equalsIgnoreCase("mysql")) + .collect(Collectors.toList()); + } + + private List getShowTableNames(String db, String catalog) throws Exception { + rootCtx.setThreadLocalInfo(); + ShowTableCommand cmd = new ShowTableCommand(db, catalog, false, PlanType.SHOW_TABLES); + ShowResultSet resultSet = cmd.doRun(rootCtx, new StmtExecutor(rootCtx, "")); + return resultSet.getResultRows().stream() + .map(row -> row.get(0)) + .collect(Collectors.toList()); + } + + private List getShowTableNamesLike(String db, String catalog, String pattern) throws Exception { + rootCtx.setThreadLocalInfo(); + ShowTableCommand cmd = new ShowTableCommand(db, catalog, false, pattern, null, PlanType.SHOW_TABLES); + ShowResultSet resultSet = cmd.doRun(rootCtx, new StmtExecutor(rootCtx, "")); + return resultSet.getResultRows().stream() + .map(row -> row.get(0)) + .collect(Collectors.toList()); + } + + private String getPlanOrError(String sql) throws Exception { + rootCtx.setThreadLocalInfo(); + return getSQLPlanOrErrorMsg(rootCtx, "EXPLAIN " + sql, false); + } + + private boolean hasNameResolutionError(String result) { + return result.contains("does not exist") + || result.contains("Unknown column") + || result.contains("unknown qualifier") + || result.contains("unknown database"); + } + + private void assertSqlSuccess(String sql) throws Exception { + String result = getPlanOrError(sql); + Assertions.assertFalse(hasNameResolutionError(result), + "Expected success but got name resolution error: " + result); + } + + private void assertSqlError(String sql) throws Exception { + String result = getPlanOrError(sql); + Assertions.assertTrue(hasNameResolutionError(result), + "Expected name resolution error but got: " + result); + } + + // ==================== SHOW DATABASES tests ==================== + + @Test + public void testShowDatabasesMode0() throws Exception { + List dbs = getShowDatabaseNames("test_mode0"); + Assertions.assertTrue(dbs.contains("DB_UPPER"), "Expected DB_UPPER, got: " + dbs); + Assertions.assertTrue(dbs.contains("MixedDb"), "Expected MixedDb, got: " + dbs); + } + + @Test + public void testShowDatabasesMode1() throws Exception { + List dbs = getShowDatabaseNames("test_mode1"); + Assertions.assertTrue(dbs.contains("db_upper"), "Expected db_upper, got: " + dbs); + Assertions.assertTrue(dbs.contains("mixeddb"), "Expected mixeddb, got: " + dbs); + } + + @Test + public void testShowDatabasesMode2() throws Exception { + List dbs = getShowDatabaseNames("test_mode2"); + Assertions.assertTrue(dbs.contains("DB_UPPER"), "Expected DB_UPPER, got: " + dbs); + Assertions.assertTrue(dbs.contains("MixedDb"), "Expected MixedDb, got: " + dbs); + } + + // ==================== SHOW TABLES tests ==================== + + @Test + public void testShowTablesMode0() throws Exception { + List tables = getShowTableNames("DB_UPPER", "test_mode0"); + Assertions.assertTrue(tables.contains("TBL_UPPER"), "Expected TBL_UPPER, got: " + tables); + Assertions.assertTrue(tables.contains("MixedTbl"), "Expected MixedTbl, got: " + tables); + } + + @Test + public void testShowTablesMode1() throws Exception { + List tables = getShowTableNames("db_upper", "test_mode1"); + Assertions.assertTrue(tables.contains("tbl_upper"), "Expected tbl_upper, got: " + tables); + Assertions.assertTrue(tables.contains("mixedtbl"), "Expected mixedtbl, got: " + tables); + } + + @Test + public void testShowTablesMode2() throws Exception { + List tables = getShowTableNames("db_upper", "test_mode2"); + Assertions.assertTrue(tables.contains("TBL_UPPER"), "Expected TBL_UPPER, got: " + tables); + Assertions.assertTrue(tables.contains("MixedTbl"), "Expected MixedTbl, got: " + tables); + } + + // ==================== SHOW TABLES DB case-sensitivity tests ==================== + + @Test + public void testShowTablesMode0WrongDbCase() { + Assertions.assertThrows(Exception.class, () -> { + getShowTableNames("db_upper", "test_mode0"); + }); + } + + @Test + public void testShowTablesMode1AnyDbCase() throws Exception { + // Mode 1 lowercases input, so DB_UPPER -> db_upper which exists + List tables = getShowTableNames("DB_UPPER", "test_mode1"); + Assertions.assertTrue(tables.contains("tbl_upper"), "Expected tbl_upper, got: " + tables); + Assertions.assertTrue(tables.contains("mixedtbl"), "Expected mixedtbl, got: " + tables); + } + + @Test + public void testShowTablesMode2AnyDbCase() throws Exception { + // Mode 2 does case-insensitive lookup, so Db_Upper finds DB_UPPER + List tables = getShowTableNames("Db_Upper", "test_mode2"); + Assertions.assertTrue(tables.contains("TBL_UPPER"), "Expected TBL_UPPER, got: " + tables); + Assertions.assertTrue(tables.contains("MixedTbl"), "Expected MixedTbl, got: " + tables); + } + + // ==================== SHOW TABLES LIKE tests ==================== + + @Test + public void testShowTablesLikeMode0CaseSensitive() throws Exception { + // Mode 0: LIKE is case-sensitive, "tbl%" should NOT match "TBL_UPPER" + List tables = getShowTableNamesLike("DB_UPPER", "test_mode0", "tbl%"); + Assertions.assertEquals(0, tables.size(), "Mode 0 LIKE should be case-sensitive, got: " + tables); + } + + @Test + public void testShowTablesLikeMode0CaseSensitiveMatch() throws Exception { + List tables = getShowTableNamesLike("DB_UPPER", "test_mode0", "TBL%"); + Assertions.assertEquals(1, tables.size(), "Expected 1 match for TBL%, got: " + tables); + Assertions.assertTrue(tables.contains("TBL_UPPER")); + } + + @Test + public void testShowTablesLikeMode1CaseInsensitive() throws Exception { + // Mode 1: LIKE is case-insensitive, "TBL%" matches "tbl_upper" + List tables = getShowTableNamesLike("db_upper", "test_mode1", "TBL%"); + Assertions.assertEquals(1, tables.size(), "Expected 1 match for TBL%, got: " + tables); + Assertions.assertTrue(tables.contains("tbl_upper")); + } + + @Test + public void testShowTablesLikeMode2CaseInsensitive() throws Exception { + // Mode 2: LIKE is case-insensitive, "tbl%" matches "TBL_UPPER" + List tables = getShowTableNamesLike("db_upper", "test_mode2", "tbl%"); + Assertions.assertEquals(1, tables.size(), "Expected 1 match for tbl%, got: " + tables); + Assertions.assertTrue(tables.contains("TBL_UPPER")); + } + + // ==================== SELECT tests — FROM clause ==================== + + @Test + public void testSelectExactCaseMode0() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectWrongTableCaseMode0() throws Exception { + assertSqlError("SELECT * FROM test_mode0.DB_UPPER.tbl_upper"); + } + + @Test + public void testSelectWrongDbCaseMode0() throws Exception { + assertSqlError("SELECT * FROM test_mode0.db_upper.TBL_UPPER"); + } + + @Test + public void testSelectAnyCaseMode1() throws Exception { + // Mode 1: DB_UPPER -> db_upper, TBL_UPPER -> tbl_upper (both exist in lowercase provider) + assertSqlSuccess("SELECT * FROM test_mode1.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectLowerCaseMode1() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode1.db_upper.tbl_upper"); + } + + @Test + public void testSelectAnyCaseMode2() throws Exception { + // Mode 2: db_upper finds DB_UPPER, tbl_upper finds TBL_UPPER + assertSqlSuccess("SELECT * FROM test_mode2.db_upper.tbl_upper"); + } + + @Test + public void testSelectMixedCaseMode2() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode2.Db_Upper.Tbl_Upper"); + } + + // ==================== A. SELECT list — qualified column references ==================== + + @Test + public void testSelectListTableQualMode0() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.id, TBL_UPPER.name FROM test_mode0.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectListTableQualMode0WrongCase() throws Exception { + assertSqlError("SELECT tbl_upper.id FROM test_mode0.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectListTableQualMode1() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.id FROM test_mode1.db_upper.tbl_upper"); + } + + @Test + public void testSelectListTableQualMode2() throws Exception { + assertSqlSuccess("SELECT tbl_upper.id FROM test_mode2.db_upper.TBL_UPPER"); + } + + // ==================== B. SELECT list — qualified star ==================== + + @Test + public void testSelectStarTableQualMode0() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.* FROM test_mode0.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectStarTableQualMode0WrongCase() throws Exception { + assertSqlError("SELECT tbl_upper.* FROM test_mode0.DB_UPPER.TBL_UPPER"); + } + + @Test + public void testSelectStarTableQualMode1() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.* FROM test_mode1.db_upper.tbl_upper"); + } + + @Test + public void testSelectStarDbTableQualMode1() throws Exception { + assertSqlSuccess("SELECT DB_UPPER.TBL_UPPER.* FROM test_mode1.db_upper.tbl_upper"); + } + + @Test + public void testSelectStarTableQualMode2() throws Exception { + assertSqlSuccess("SELECT tbl_upper.* FROM test_mode2.db_upper.TBL_UPPER"); + } + + // ==================== C. WHERE clause — filter predicates ==================== + + @Test + public void testWhereTableQualMode0() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER WHERE TBL_UPPER.id = 1"); + } + + @Test + public void testWhereTableQualMode0WrongCase() throws Exception { + assertSqlError("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER WHERE tbl_upper.id = 1"); + } + + @Test + public void testWhereTableQualMode1() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode1.db_upper.tbl_upper WHERE TBL_UPPER.id = 1"); + } + + @Test + public void testWhereTableQualMode2() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode2.db_upper.TBL_UPPER WHERE tbl_upper.id = 1"); + } + + @Test + public void testWhereDbTableQualMode0() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER WHERE DB_UPPER.TBL_UPPER.id = 1"); + } + + @Test + public void testWhereDbTableQualMode0WrongCase() throws Exception { + assertSqlError("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER WHERE db_upper.tbl_upper.id = 1"); + } + + @Test + public void testWhereDbTableQualMode1() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode1.db_upper.tbl_upper WHERE DB_UPPER.TBL_UPPER.id = 1"); + } + + @Test + public void testWhereDbTableQualMode2() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode2.db_upper.TBL_UPPER WHERE db_upper.tbl_upper.id = 1"); + } + + // ==================== D. JOIN — table reference + ON clause qualifiers ==================== + + @Test + public void testJoinMode0() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.id FROM test_mode0.DB_UPPER.TBL_UPPER " + + "JOIN test_mode0.DB_UPPER.MixedTbl ON TBL_UPPER.id = MixedTbl.k1"); + } + + @Test + public void testJoinMode0WrongCase() throws Exception { + assertSqlError("SELECT tbl_upper.id FROM test_mode0.DB_UPPER.TBL_UPPER " + + "JOIN test_mode0.DB_UPPER.MixedTbl ON tbl_upper.id = mixedtbl.k1"); + } + + @Test + public void testJoinMode1() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.id FROM test_mode1.db_upper.tbl_upper " + + "JOIN test_mode1.db_upper.mixedtbl ON TBL_UPPER.id = MixedTbl.k1"); + } + + @Test + public void testJoinMode2() throws Exception { + assertSqlSuccess("SELECT tbl_upper.id FROM test_mode2.db_upper.TBL_UPPER " + + "JOIN test_mode2.db_upper.MixedTbl ON tbl_upper.id = mixedtbl.k1"); + } + + // ==================== E. GROUP BY — qualified column reference ==================== + + @Test + public void testGroupByMode0() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.name, count(*) FROM test_mode0.DB_UPPER.TBL_UPPER " + + "GROUP BY TBL_UPPER.name"); + } + + @Test + public void testGroupByMode0WrongCase() throws Exception { + assertSqlError("SELECT TBL_UPPER.name, count(*) FROM test_mode0.DB_UPPER.TBL_UPPER " + + "GROUP BY tbl_upper.name"); + } + + @Test + public void testGroupByMode1() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.name, count(*) FROM test_mode1.db_upper.tbl_upper " + + "GROUP BY TBL_UPPER.name"); + } + + @Test + public void testGroupByMode2() throws Exception { + assertSqlSuccess("SELECT tbl_upper.name, count(*) FROM test_mode2.db_upper.TBL_UPPER " + + "GROUP BY tbl_upper.name"); + } + + // ==================== F. ORDER BY — qualified column reference ==================== + + @Test + public void testOrderByMode0() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER ORDER BY TBL_UPPER.id"); + } + + @Test + public void testOrderByMode0WrongCase() throws Exception { + assertSqlError("SELECT * FROM test_mode0.DB_UPPER.TBL_UPPER ORDER BY tbl_upper.id"); + } + + @Test + public void testOrderByMode1() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode1.db_upper.tbl_upper ORDER BY TBL_UPPER.id"); + } + + @Test + public void testOrderByMode2() throws Exception { + assertSqlSuccess("SELECT * FROM test_mode2.db_upper.TBL_UPPER ORDER BY tbl_upper.id"); + } + + // ==================== G. HAVING — qualified column in aggregate filter ==================== + + @Test + public void testHavingMode0() throws Exception { + assertSqlSuccess("SELECT TBL_UPPER.name FROM test_mode0.DB_UPPER.TBL_UPPER " + + "GROUP BY TBL_UPPER.name HAVING count(TBL_UPPER.id) > 0"); + } + + @Test + public void testHavingMode0WrongCase() throws Exception { + assertSqlError("SELECT TBL_UPPER.name FROM test_mode0.DB_UPPER.TBL_UPPER " + + "GROUP BY TBL_UPPER.name HAVING count(tbl_upper.id) > 0"); + } + + @Test + public void testHavingMode1() throws Exception { + assertSqlSuccess("SELECT tbl_upper.name FROM test_mode1.db_upper.tbl_upper " + + "GROUP BY tbl_upper.name HAVING count(TBL_UPPER.id) > 0"); + } + + // ==================== H. Subquery — table reference inside subquery ==================== + + @Test + public void testSubqueryMode0() throws Exception { + assertSqlSuccess("SELECT * FROM (SELECT TBL_UPPER.id FROM test_mode0.DB_UPPER.TBL_UPPER) t"); + } + + @Test + public void testSubqueryMode0WrongCase() throws Exception { + assertSqlError("SELECT * FROM (SELECT id FROM test_mode0.DB_UPPER.tbl_upper) t"); + } + + @Test + public void testSubqueryMode1() throws Exception { + assertSqlSuccess("SELECT * FROM (SELECT id FROM test_mode1.DB_UPPER.TBL_UPPER) t"); + } + + @Test + public void testSubqueryMode2() throws Exception { + assertSqlSuccess("SELECT * FROM (SELECT id FROM test_mode2.Db_Upper.Tbl_Upper) t"); + } + + // ==================== I. Cross-database JOIN ==================== + + @Test + public void testCrossDbJoinMode1() throws Exception { + assertSqlSuccess("SELECT a.id FROM test_mode1.DB_UPPER.TBL_UPPER a " + + "JOIN test_mode1.MIXEDDB.another_tbl b ON a.id = b.x"); + } + + @Test + public void testCrossDbJoinMode2() throws Exception { + assertSqlSuccess("SELECT a.id FROM test_mode2.db_upper.tbl_upper a " + + "JOIN test_mode2.mixeddb.another_tbl b ON a.id = b.x"); + } + + // ==================== Mock data providers ==================== + + /** + * Mixed-case provider for mode 0 (case-sensitive) and mode 2 (case-insensitive comparison). + * Remote names preserve original casing. + */ + public static class MixedCaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + + Map> dbUpperTables = Maps.newHashMap(); + dbUpperTables.put("TBL_UPPER", Lists.newArrayList( + new Column("id", PrimitiveType.INT), + new Column("name", PrimitiveType.VARCHAR))); + dbUpperTables.put("MixedTbl", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR))); + MOCKED_META.put("DB_UPPER", dbUpperTables); + + Map> mixedDbTables = Maps.newHashMap(); + mixedDbTables.put("another_tbl", Lists.newArrayList( + new Column("x", PrimitiveType.INT), + new Column("y", PrimitiveType.VARCHAR))); + MOCKED_META.put("MixedDb", mixedDbTables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } + + /** + * Lowercase provider for mode 1 (stored as lowercase). + * In real-world mode 1, the remote system stores names in lowercase. + * The test verifies that any-case SQL input gets lowered and resolves correctly. + */ + public static class LowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + + Map> dbUpperTables = Maps.newHashMap(); + dbUpperTables.put("tbl_upper", Lists.newArrayList( + new Column("id", PrimitiveType.INT), + new Column("name", PrimitiveType.VARCHAR))); + dbUpperTables.put("mixedtbl", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR))); + MOCKED_META.put("db_upper", dbUpperTables); + + Map> mixedDbTables = Maps.newHashMap(); + mixedDbTables.put("another_tbl", Lists.newArrayList( + new Column("x", PrimitiveType.INT), + new Column("y", PrimitiveType.VARCHAR))); + MOCKED_META.put("mixeddb", mixedDbTables); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out b/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out index 5ce72113334eb8..32628a0578083f 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out +++ b/regression-test/data/external_table_p0/hive/test_hive_case_sensibility.out @@ -101,6 +101,7 @@ case_db1 -- !sql7 -- -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 @@ -195,6 +196,7 @@ case_db1 -- !sql7 -- -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out index 953e628f309658..7b68204a7504ba 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out @@ -106,8 +106,10 @@ iceberg_hadoop_case_db1 -- !sql7 -- -- !sqlx -- +case_tbl11 -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 @@ -205,11 +207,13 @@ iceberg_hadoop_case_db1 -- !sql7 -- -- !sqlx -- +case_tbl11 -- !sql8 -- CASE_TBL14 -- !sql9 -- +CASE_TBL14 -- !sql10 -- case_tbl13 @@ -219,6 +223,7 @@ case_tbl13 -- !sql12 -- -- !sql12 -- +CASE_TBL22 -- !sql13 -- test_iceberg_case_sensibility_hadoop iceberg_hadoop_case_db2 CASE_TBL22 BASE TABLE iceberg \N \N -1 0 0 \N 0 \N \N \N \N \N utf-8 \N \N diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.out index a3a1d68e7e2aa2..d70c1966c14e24 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.out @@ -90,6 +90,7 @@ iceberg_hms_case_db1 -- !sql7 -- -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 @@ -173,6 +174,7 @@ iceberg_hms_case_db1 -- !sql7 -- -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out index c6379666096100..52f66385338a8b 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out @@ -106,8 +106,10 @@ iceberg_rest_case_db1 -- !sql7 -- -- !sqlx -- +case_tbl11 -- !sql8 -- +case_tbl14 -- !sql9 -- case_tbl14 @@ -205,11 +207,13 @@ iceberg_rest_case_db1 -- !sql7 -- -- !sqlx -- +case_tbl11 -- !sql8 -- CASE_TBL14 -- !sql9 -- +CASE_TBL14 -- !sql10 -- case_tbl13 @@ -219,6 +223,7 @@ case_tbl13 -- !sql12 -- -- !sql12 -- +CASE_TBL22 -- !sql13 -- test_iceberg_case_sensibility_rest iceberg_rest_case_db2 CASE_TBL22 BASE TABLE iceberg \N \N -1 0 0 \N 0 \N \N \N \N \N utf-8 \N \N diff --git a/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy b/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy index bdf8ee094ed6e5..1d00df13ee2dbf 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_case_sensibility.groovy @@ -130,7 +130,7 @@ suite("test_hive_case_sensibility", "p0,external,doris,external_docker,external_ sql """create table case_tbl13 (k1 int);""" sql """create table CASE_TBL14 (k1 int);""" - qt_sql8 """show tables like "%CASE_TBL14%"""" // empty + qt_sql8 """show tables like "%CASE_TBL14%"""" // not empty, because lower=1 qt_sql9 """show tables like "%case_tbl14%"""" qt_sql10 """show tables like "%case_tbl13%"""" diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy index 26a35b86cee1a0..2517739eecd8c4 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy @@ -125,6 +125,7 @@ suite("test_iceberg_hadoop_case_sensibility", "p0,external,doris,external_docker exception "Table 'CASE_TBL11' already exists" } } + // for case 1,2, like is case insensible, and CASE_TBL11 is not created, so will show case_tbl11 qt_sqlx """show tables from iceberg_hadoop_case_db1 like "%CASE_TBL11%"""" sql """create table iceberg_hadoop_case_db1.CASE_TBL12 (k1 int);""" @@ -133,7 +134,7 @@ suite("test_iceberg_hadoop_case_sensibility", "p0,external,doris,external_docker sql """create table CASE_TBL14 (k1 int);""" qt_sql8 """show tables like "%CASE_TBL14%"""" - qt_sql9 """show tables like "%case_tbl14%"""" // empty + qt_sql9 """show tables like "%case_tbl14%"""" qt_sql10 """show tables like "%case_tbl13%"""" test { @@ -142,6 +143,7 @@ suite("test_iceberg_hadoop_case_sensibility", "p0,external,doris,external_docker } qt_sql11 """show tables from iceberg_hadoop_case_db2 like "%case_tbl14%"""" // empty qt_sql12 """show tables from iceberg_hadoop_case_db2 like "%case_tbl21%"""" // empty + // for case 1,2, like is case insensible, and case_tbl22 is not created, so will show CASE_TBL22 qt_sql12 """show tables from iceberg_hadoop_case_db2 like "%case_tbl22%"""" order_qt_sql13 """select * from information_schema.tables where TABLE_SCHEMA="iceberg_hadoop_case_db2";""" diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.groovy index 01a10c22ebfad1..9e87b119926925 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_hms_case_sensibility.groovy @@ -131,7 +131,7 @@ suite("test_iceberg_hms_case_sensibility", "p0,external,doris,external_docker,ex sql """create table case_tbl13 (k1 int);""" sql """create table CASE_TBL14 (k1 int);""" - qt_sql8 """show tables like "%CASE_TBL14%"""" // empty + qt_sql8 """show tables like "%CASE_TBL14%"""" // not empty, because lower=1 qt_sql9 """show tables like "%case_tbl14%"""" qt_sql10 """show tables like "%case_tbl13%"""" diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy index 510b2c00362a2c..834dde0cfd8422 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy @@ -126,6 +126,7 @@ suite("test_iceberg_rest_case_sensibility", "p0,external,doris,external_docker,e exception "Table 'CASE_TBL11' already exists" } } + // for case 1,2, like is case insensible, and CASE_TBL11 is not created, so will show case_tbl11 qt_sqlx """show tables from iceberg_rest_case_db1 like "%CASE_TBL11%"""" sql """create table iceberg_rest_case_db1.CASE_TBL12 (k1 int);""" @@ -134,7 +135,7 @@ suite("test_iceberg_rest_case_sensibility", "p0,external,doris,external_docker,e sql """create table CASE_TBL14 (k1 int);""" qt_sql8 """show tables like "%CASE_TBL14%"""" - qt_sql9 """show tables like "%case_tbl14%"""" // empty + qt_sql9 """show tables like "%case_tbl14%"""" qt_sql10 """show tables like "%case_tbl13%"""" test { @@ -143,6 +144,7 @@ suite("test_iceberg_rest_case_sensibility", "p0,external,doris,external_docker,e } qt_sql11 """show tables from iceberg_rest_case_db2 like "%case_tbl14%"""" // empty qt_sql12 """show tables from iceberg_rest_case_db2 like "%case_tbl21%"""" // empty + // for case 1,2, like is case insensible, and case_tbl22 is not created, so will show CASE_TBL22 qt_sql12 """show tables from iceberg_rest_case_db2 like "%case_tbl22%"""" order_qt_sql13 """select * from information_schema.tables where TABLE_SCHEMA="iceberg_rest_case_db2";""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy index 1c218457e6f2b4..87a0bf27e50665 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy @@ -27,6 +27,7 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external String s3_endpoint = getS3Endpoint() String bucket = getS3BucketName() String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + // String driver_url = "mysql-connector-j-8.4.0.jar" def wait_table_sync = { String db -> Awaitility.await().atMost(10, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until{ @@ -414,7 +415,7 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external // Verification results include lower and UPPER check { result, ex, startTime, endTime -> - def expectedTables = ["lower_with_conf", "upper_with_conf"] + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] expectedTables.each { tableName -> assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") } @@ -519,7 +520,7 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external // Verification results include lower and UPPER check { result, ex, startTime, endTime -> - def expectedTables = ["lower_with_conf", "upper_with_conf"] + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] expectedTables.each { tableName -> assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") }