Skip to content

Commit 71c5f35

Browse files
morningmanmorningman
authored
[opt](catalog) remove old properties (#55625)
1 parent 22eb680 commit 71c5f35

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+124
-1829
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
import org.apache.doris.common.DdlException;
2121
import org.apache.doris.common.proc.BaseProcResult;
22-
import org.apache.doris.datasource.property.constants.HMSProperties;
22+
import org.apache.doris.datasource.property.metastore.HMSBaseProperties;
2323

2424
import com.google.common.collect.ImmutableMap;
2525
import com.google.common.collect.Lists;
@@ -63,10 +63,8 @@ public void modifyProperties(Map<String, String> properties) throws DdlException
6363

6464
@Override
6565
protected void setProperties(ImmutableMap<String, String> properties) throws DdlException {
66-
for (String field : HMSProperties.REQUIRED_FIELDS) {
67-
if (!properties.containsKey(field)) {
68-
throw new DdlException("Missing [" + field + "] in properties.");
69-
}
66+
if (!properties.containsKey(HMSBaseProperties.HIVE_METASTORE_URIS)) {
67+
throw new DdlException("Missing [" + HMSBaseProperties.HIVE_METASTORE_URIS + "] in properties.");
7068
}
7169
this.properties.putAll(properties);
7270
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/HiveTable.java‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020
import org.apache.doris.common.DdlException;
2121
import org.apache.doris.common.security.authentication.AuthType;
2222
import org.apache.doris.common.security.authentication.AuthenticationConfig;
23-
import org.apache.doris.datasource.property.constants.HMSProperties;
2423
import org.apache.doris.datasource.property.constants.S3Properties;
24+
import org.apache.doris.datasource.property.metastore.HMSBaseProperties;
2525
import org.apache.doris.thrift.THiveTable;
2626
import org.apache.doris.thrift.TTableDescriptor;
2727
import org.apache.doris.thrift.TTableType;
@@ -102,18 +102,19 @@ private void validate(Map<String, String> properties) throws DdlException {
102102

103103
// check hive properties
104104
// hive.metastore.uris
105-
String hiveMetaStoreUris = copiedProps.get(HMSProperties.HIVE_METASTORE_URIS);
105+
String hiveMetaStoreUris = copiedProps.get(HMSBaseProperties.HIVE_METASTORE_URIS);
106106
if (Strings.isNullOrEmpty(hiveMetaStoreUris)) {
107107
throw new DdlException(String.format(
108-
PROPERTY_MISSING_MSG, HMSProperties.HIVE_METASTORE_URIS, HMSProperties.HIVE_METASTORE_URIS));
108+
PROPERTY_MISSING_MSG, HMSBaseProperties.HIVE_METASTORE_URIS,
109+
HMSBaseProperties.HIVE_METASTORE_URIS));
109110
}
110-
copiedProps.remove(HMSProperties.HIVE_METASTORE_URIS);
111-
hiveProperties.put(HMSProperties.HIVE_METASTORE_URIS, hiveMetaStoreUris);
111+
copiedProps.remove(HMSBaseProperties.HIVE_METASTORE_URIS);
112+
hiveProperties.put(HMSBaseProperties.HIVE_METASTORE_URIS, hiveMetaStoreUris);
112113
// support multi hive version
113-
String hiveVersion = copiedProps.get(HMSProperties.HIVE_VERSION);
114+
String hiveVersion = copiedProps.get(HMSBaseProperties.HIVE_VERSION);
114115
if (!Strings.isNullOrEmpty(hiveVersion)) {
115-
copiedProps.remove(HMSProperties.HIVE_VERSION);
116-
hiveProperties.put(HMSProperties.HIVE_VERSION, hiveVersion);
116+
copiedProps.remove(HMSBaseProperties.HIVE_VERSION);
117+
hiveProperties.put(HMSBaseProperties.HIVE_VERSION, hiveVersion);
117118
}
118119

119120
// check auth type

‎fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919

2020
import org.apache.doris.common.DdlException;
2121
import org.apache.doris.common.UserException;
22-
import org.apache.doris.datasource.property.PropertyConverter;
2322
import org.apache.doris.datasource.property.constants.S3Properties;
2423
import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand;
2524

@@ -64,7 +63,7 @@ public class S3StorageVault extends StorageVault {
6463
public static class PropertyKey {
6564
public static final String ACCESS_KEY = S3Properties.ACCESS_KEY;
6665
public static final String SECRET_KEY = S3Properties.SECRET_KEY;
67-
public static final String USE_PATH_STYLE = PropertyConverter.USE_PATH_STYLE;
66+
public static final String USE_PATH_STYLE = S3Properties.USE_PATH_STYLE;
6867
public static final String ROOT_PATH = S3Properties.ROOT_PATH;
6968
public static final String PROVIDER = S3Properties.PROVIDER;
7069
public static final String REGION = S3Properties.REGION;

‎fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,17 @@
1818
package org.apache.doris.common.util;
1919

2020
import org.apache.doris.datasource.property.ConnectorPropertiesUtils;
21-
import org.apache.doris.datasource.property.constants.CosProperties;
22-
import org.apache.doris.datasource.property.constants.GCSProperties;
23-
import org.apache.doris.datasource.property.constants.GlueProperties;
2421
import org.apache.doris.datasource.property.constants.MCProperties;
25-
import org.apache.doris.datasource.property.constants.ObsProperties;
26-
import org.apache.doris.datasource.property.constants.OssProperties;
2722
import org.apache.doris.datasource.property.constants.S3Properties;
23+
import org.apache.doris.datasource.property.metastore.AWSGlueMetaStoreBaseProperties;
2824
import org.apache.doris.datasource.property.metastore.AliyunDLFBaseProperties;
25+
import org.apache.doris.datasource.property.storage.AzureProperties;
26+
import org.apache.doris.datasource.property.storage.COSProperties;
27+
import org.apache.doris.datasource.property.storage.GCSProperties;
28+
import org.apache.doris.datasource.property.storage.MinioProperties;
29+
import org.apache.doris.datasource.property.storage.OBSProperties;
30+
import org.apache.doris.datasource.property.storage.OSSHdfsProperties;
31+
import org.apache.doris.datasource.property.storage.OSSProperties;
2932

3033
import com.google.common.collect.Sets;
3134

@@ -60,18 +63,20 @@ public class PrintableMap<K, V> {
6063
SENSITIVE_KEY.addAll(Arrays.asList(
6164
S3Properties.SECRET_KEY,
6265
S3Properties.Env.SECRET_KEY,
63-
ObsProperties.SECRET_KEY,
64-
OssProperties.SECRET_KEY,
65-
GCSProperties.SECRET_KEY,
66-
CosProperties.SECRET_KEY,
67-
GlueProperties.SECRET_KEY,
6866
MCProperties.SECRET_KEY));
6967

7068
SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(AliyunDLFBaseProperties.class));
69+
SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(AWSGlueMetaStoreBaseProperties.class));
70+
SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(GCSProperties.class));
71+
SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(AzureProperties.class));
72+
SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(OSSProperties.class));
73+
SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(OSSHdfsProperties.class));
74+
SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(COSProperties.class));
75+
SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(OBSProperties.class));
76+
SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(MinioProperties.class));
7177

7278
HIDDEN_KEY = Sets.newHashSet();
7379
HIDDEN_KEY.addAll(S3Properties.Env.FS_KEYS);
74-
HIDDEN_KEY.addAll(GlueProperties.META_KEYS);
7580
}
7681

7782
public PrintableMap(Map<K, V> map, String keyValueSeparator,

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
import org.apache.doris.datasource.ExternalTable;
3535
import org.apache.doris.datasource.NameMapping;
3636
import org.apache.doris.datasource.operations.ExternalMetadataOps;
37-
import org.apache.doris.datasource.property.constants.HMSProperties;
37+
import org.apache.doris.datasource.property.metastore.HMSBaseProperties;
3838
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo;
3939
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo;
4040
import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo;
@@ -237,8 +237,8 @@ In hive, this table only can insert not update(not report error,but not actually
237237

238238
}
239239
Map<String, String> properties = catalog.getProperties();
240-
if (properties.containsKey(HMSProperties.HIVE_METASTORE_TYPE)
241-
&& properties.get(HMSProperties.HIVE_METASTORE_TYPE).equals(HMSProperties.DLF_TYPE)) {
240+
if (properties.containsKey(HMSBaseProperties.HIVE_METASTORE_TYPE)
241+
&& properties.get(HMSBaseProperties.HIVE_METASTORE_TYPE).equals(HMSBaseProperties.DLF_TYPE)) {
242242
for (Column column : createTableInfo.getColumns()) {
243243
if (column.hasDefaultValue()) {
244244
throw new UserException("Default values are not supported with `DLF` catalog.");
@@ -352,8 +352,8 @@ In hive, this table only can insert not update(not report error,but not actually
352352

353353
}
354354
Map<String, String> properties = catalog.getProperties();
355-
if (properties.containsKey(HMSProperties.HIVE_METASTORE_TYPE)
356-
&& properties.get(HMSProperties.HIVE_METASTORE_TYPE).equals(HMSProperties.DLF_TYPE)) {
355+
if (properties.containsKey(HMSBaseProperties.HIVE_METASTORE_TYPE)
356+
&& properties.get(HMSBaseProperties.HIVE_METASTORE_TYPE).equals(HMSBaseProperties.DLF_TYPE)) {
357357
for (Column column : stmt.getColumns()) {
358358
if (column.hasDefaultValue()) {
359359
throw new UserException("Default values are not supported with `DLF` catalog.");

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import org.apache.doris.datasource.DatabaseMetadata;
2525
import org.apache.doris.datasource.TableMetadata;
2626
import org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException;
27-
import org.apache.doris.datasource.property.constants.HMSProperties;
27+
import org.apache.doris.datasource.property.metastore.HMSBaseProperties;
2828

2929
import com.aliyun.datalake.metastore.hive2.ProxyMetaStoreClient;
3030
import com.amazonaws.glue.catalog.metastore.AWSCatalogMetastoreClient;
@@ -646,11 +646,11 @@ private class ThriftHMSClient implements AutoCloseable {
646646
private volatile Throwable throwable;
647647

648648
private ThriftHMSClient(HiveConf hiveConf) throws MetaException {
649-
String type = hiveConf.get(HMSProperties.HIVE_METASTORE_TYPE);
650-
if (HMSProperties.DLF_TYPE.equalsIgnoreCase(type)) {
649+
String type = hiveConf.get(HMSBaseProperties.HIVE_METASTORE_TYPE);
650+
if (HMSBaseProperties.DLF_TYPE.equalsIgnoreCase(type)) {
651651
client = RetryingMetaStoreClient.getProxy(hiveConf, DUMMY_HOOK_LOADER,
652652
ProxyMetaStoreClient.class.getName());
653-
} else if (HMSProperties.GLUE_TYPE.equalsIgnoreCase(type)) {
653+
} else if (HMSBaseProperties.GLUE_TYPE.equalsIgnoreCase(type)) {
654654
client = RetryingMetaStoreClient.getProxy(hiveConf, DUMMY_HOOK_LOADER,
655655
AWSCatalogMetastoreClient.class.getName());
656656
} else {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import org.apache.doris.common.DdlException;
2323
import org.apache.doris.common.UserException;
2424
import org.apache.doris.datasource.CatalogProperty;
25-
import org.apache.doris.datasource.property.constants.HMSProperties;
25+
import org.apache.doris.datasource.property.metastore.HMSBaseProperties;
2626
import org.apache.doris.nereids.exceptions.NotSupportedException;
2727
import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
2828

@@ -33,7 +33,7 @@ public class IcebergDLFExternalCatalog extends IcebergExternalCatalog {
3333
public IcebergDLFExternalCatalog(long catalogId, String name, String resource, Map<String, String> props,
3434
String comment) {
3535
super(catalogId, name, comment);
36-
props.put(HMSProperties.HIVE_METASTORE_TYPE, "dlf");
36+
props.put(HMSBaseProperties.HIVE_METASTORE_TYPE, "dlf");
3737
catalogProperty = new CatalogProperty(resource, props);
3838
}
3939

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo;
6060
import org.apache.doris.datasource.mvcc.MvccSnapshot;
6161
import org.apache.doris.datasource.mvcc.MvccUtil;
62-
import org.apache.doris.datasource.property.constants.HMSProperties;
62+
import org.apache.doris.datasource.property.metastore.HMSBaseProperties;
6363
import org.apache.doris.nereids.exceptions.NotSupportedException;
6464
import org.apache.doris.thrift.TExprOpcode;
6565

@@ -937,7 +937,7 @@ public static HiveCatalog createIcebergHiveCatalog(ExternalCatalog externalCatal
937937
// Later, type checks will be performed when loading the table.
938938
catalogProperties.put(HiveCatalog.LIST_ALL_TABLES, "true");
939939
}
940-
String metastoreUris = catalogProperties.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, "");
940+
String metastoreUris = catalogProperties.getOrDefault(HMSBaseProperties.HIVE_METASTORE_URIS, "");
941941
catalogProperties.put(CatalogProperties.URI, metastoreUris);
942942
hiveCatalog.initialize(name, catalogProperties);
943943
return hiveCatalog;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import org.apache.doris.datasource.TableFormatType;
2626
import org.apache.doris.datasource.lakesoul.LakeSoulExternalTable;
2727
import org.apache.doris.datasource.lakesoul.LakeSoulUtils;
28-
import org.apache.doris.datasource.property.constants.OssProperties;
2928
import org.apache.doris.datasource.property.constants.S3Properties;
3029
import org.apache.doris.planner.PlanNodeId;
3130
import org.apache.doris.qe.SessionVariable;
@@ -183,7 +182,7 @@ private void setLakeSoulParams(TFileRangeDesc rangeDesc, LakeSoulSplit lakeSoulS
183182

184183
if (catalogProps.get(S3Properties.Env.ENDPOINT) != null) {
185184
options.put(LakeSoulUtils.FS_S3A_ENDPOINT, catalogProps.get(S3Properties.Env.ENDPOINT));
186-
if (!options.containsKey(OssProperties.ENDPOINT)) {
185+
if (!options.containsKey("oss.endpoint")) {
187186
// Aliyun OSS requires virtual host style access
188187
options.put(LakeSoulUtils.FS_S3A_PATH_STYLE_ACCESS, "false");
189188
} else {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.apache.doris.datasource.ExternalCatalog;
2424
import org.apache.doris.datasource.mvcc.MvccUtil;
2525
import org.apache.doris.datasource.paimon.PaimonExternalTable;
26-
import org.apache.doris.datasource.property.constants.PaimonProperties;
2726
import org.apache.doris.thrift.TFileAttributes;
2827

2928
import com.google.common.annotations.VisibleForTesting;
@@ -69,6 +68,6 @@ public ExternalCatalog getCatalog() {
6968
}
7069

7170
public String getFileFormatFromTableProperties() {
72-
return originTable.options().getOrDefault(PaimonProperties.FILE_FORMAT, "parquet");
71+
return originTable.options().getOrDefault("file.format", "parquet");
7372
}
7473
}

0 commit comments

Comments
 (0)