From c5130514621a3ccef1f56b5e8ab5be8cb76a49ef Mon Sep 17 00:00:00 2001 From: qiye Date: Mon, 13 May 2024 15:27:51 +0800 Subject: [PATCH] [improvement](inverted index) Disable the use of skipping write index on load (#34719) When `skip_write_index_on_load` is turned on, users will get an error when querying for the latest data(not compacted), giving them a bad experience. And we can use `inverted_index_ram_dir_enable = true` and `inverted_index_storage_format=V2` to reduce IO and CPU consumption. So we disable it now. 1. Disable setting `skip_write_index_on_load` to `true` in create table stmt. 2. Disable setting `skip_write_index_on_load` to `true` in alter table properties stmt. You can still alter `skip_write_index_on_load` to `false`. Co-authored-by: Luennng --- .../apache/doris/analysis/ModifyTablePropertiesClause.java | 5 +++++ .../java/org/apache/doris/common/util/PropertyAnalyzer.java | 3 ++- .../java/org/apache/doris/datasource/InternalCatalog.java | 3 +-- regression-test/suites/datatype_p0/scalar_types/load.groovy | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java index e0b0914634bbcf..b6ef97815bc67f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java @@ -240,6 +240,11 @@ public void analyze(Analyzer analyzer) throws AnalysisException { this.needTableStable = false; this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) { + if (properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true")) { + throw new AnalysisException( + "Property " + + PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD + " is forbidden now"); + } if (!properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true") && !properties.get(PropertyAnalyzer .PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("false")) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 91848292f89f1b..0972018f5721e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -798,7 +798,8 @@ public static Boolean analyzeSkipWriteIndexOnLoad(Map properties } properties.remove(PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD); if (value.equalsIgnoreCase("true")) { - return true; + throw new AnalysisException("Property " + PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD + + " is forbidden now."); } else if (value.equalsIgnoreCase("false")) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 1f51f1f9c7cbeb..d85d335c45d53a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2492,8 +2492,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep olapTable.setStoreRowColumn(storeRowColumn); // set skip inverted index on load - boolean skipWriteIndexOnLoad = PropertyAnalyzer.analyzeBooleanProp(properties, - PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, false); + boolean skipWriteIndexOnLoad = PropertyAnalyzer.analyzeSkipWriteIndexOnLoad(properties); olapTable.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad); boolean isMutable = PropertyAnalyzer.analyzeBooleanProp(properties, PropertyAnalyzer.PROPERTIES_MUTABLE, true); diff --git a/regression-test/suites/datatype_p0/scalar_types/load.groovy b/regression-test/suites/datatype_p0/scalar_types/load.groovy index 91cd4eb1f1f1d6..4c7c7ee93666de 100644 --- a/regression-test/suites/datatype_p0/scalar_types/load.groovy +++ b/regression-test/suites/datatype_p0/scalar_types/load.groovy @@ -448,7 +448,7 @@ suite("test_scalar_types_load", "p0") { DUPLICATE KEY(`k1`) COMMENT 'OLAP' DISTRIBUTED BY HASH(`k1`) BUCKETS 10 - PROPERTIES("replication_num" = "1", "skip_write_index_on_load" = "true"); + PROPERTIES("replication_num" = "1", "skip_write_index_on_load" = "false"); """ // insert data into dup table with index