[improvement](inverted index) Disable the use of skipping write index…

… on load (apache#34719) When `skip_write_index_on_load` is turned on, users will get an error when querying for the latest data(not compacted), giving them a bad experience. And we can use `inverted_index_ram_dir_enable = true` and `inverted_index_storage_format=V2` to reduce IO and CPU consumption. So we disable it now. 1. Disable setting `skip_write_index_on_load` to `true` in create table stmt. 2. Disable setting `skip_write_index_on_load` to `true` in alter table properties stmt. You can still alter `skip_write_index_on_load` to `false`. Co-authored-by: Luennng <luennng@gmail.com>
ByteYue · May 15, 2024 · c513051 · c513051
1 parent aec704c
commit c513051
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 4 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -240,6 +240,11 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
             this.needTableStable = false;
             this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
         } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) {
+            if (properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true")) {
+                throw new AnalysisException(
+                    "Property "
+                    + PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD + " is forbidden now");
+            }
             if (!properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true")
                     && !properties.get(PropertyAnalyzer
                                                 .PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("false")) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -798,7 +798,8 @@ public static Boolean analyzeSkipWriteIndexOnLoad(Map<String, String> properties
         }
         properties.remove(PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD);
         if (value.equalsIgnoreCase("true")) {
-            return true;
+            throw new AnalysisException("Property " + PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD
+                    + " is forbidden now.");
         } else if (value.equalsIgnoreCase("false")) {
             return false;
         }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2492,8 +2492,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep
         olapTable.setStoreRowColumn(storeRowColumn);
 
         // set skip inverted index on load
-        boolean skipWriteIndexOnLoad = PropertyAnalyzer.analyzeBooleanProp(properties,
-                PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, false);
+        boolean skipWriteIndexOnLoad = PropertyAnalyzer.analyzeSkipWriteIndexOnLoad(properties);
         olapTable.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad);
 
         boolean isMutable = PropertyAnalyzer.analyzeBooleanProp(properties, PropertyAnalyzer.PROPERTIES_MUTABLE, true);

diff --git a/regression-test/suites/datatype_p0/scalar_types/load.groovy b/regression-test/suites/datatype_p0/scalar_types/load.groovy
@@ -448,7 +448,7 @@ suite("test_scalar_types_load", "p0") {
         DUPLICATE KEY(`k1`)
         COMMENT 'OLAP'
         DISTRIBUTED BY HASH(`k1`) BUCKETS 10
-        PROPERTIES("replication_num" = "1", "skip_write_index_on_load" = "true");
+        PROPERTIES("replication_num" = "1", "skip_write_index_on_load" = "false");
         """
 
     // insert data into dup table with index