branch-4.0: [fix](filecache) Restrict file cache ttl property changes

fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java

@@ -83,6 +83,8 @@ public void analyze() throws AnalysisException {
             throw new AnalysisException("Properties is not set");
         }
 
+        PropertyAnalyzer.analyzeTTLAlter(properties);
+
         if (properties.size() != 1
                 && !TableProperty.isSamePrefixProperties(
                         properties, DynamicPartitionProperty.DYNAMIC_PARTITION_PROPERTY_PREFIX)
@@ -348,9 +350,6 @@ public void analyze() throws AnalysisException {
             }
             this.needTableStable = false;
             this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
-        } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS)) {
-            this.needTableStable = false;
-            this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
         } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_VAULT_NAME)) {
             throw new AnalysisException("You can not modify storage vault name");
         } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_VAULT_ID)) {

fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java

@@ -98,11 +98,11 @@ public void updatePartitionsProperties(Database db, String tableName, List<Strin
     @Override
     public void updateTableProperties(Database db, String tableName, Map<String, String> properties)
             throws UserException {
+        PropertyAnalyzer.analyzeTTLAlter(properties);
         final Set<String> allowedProps = new HashSet<String>() {
             {
                 add(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS);
                 add(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_DATA_BYTES);
-                add(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS);
                 add(PropertyAnalyzer.PROPERTIES_COMPACTION_POLICY);
                 add(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES);
                 add(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_FILE_COUNT_THRESHOLD);
@@ -136,22 +136,7 @@ public void updateTableProperties(Database db, String tableName, Map<String, Str
                     + PropertyAnalyzer.PROPERTIES_PARTITION_RETENTION_COUNT);
         }
 
-        if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS)) {
-            long ttlSeconds = PropertyAnalyzer.analyzeTTL(properties);
-            olapTable.readLock();
-            try {
-                if (ttlSeconds == olapTable.getTTLSeconds()) {
-                    LOG.info("ttlSeconds:{} is equal with olapTable.getTTLSeconds():{}", ttlSeconds,
-                            olapTable.getTTLSeconds());
-                    return;
-                }
-                partitions.addAll(olapTable.getPartitions());
-            } finally {
-                olapTable.readUnlock();
-            }
-            param.ttlSeconds = ttlSeconds;
-            param.type = UpdatePartitionMetaParam.TabletMetaType.TTL_SECONDS;
-        } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS)) {
+        if (properties.containsKey(PropertyAnalyzer.PROPERTIES_GROUP_COMMIT_INTERVAL_MS)) {
             long groupCommitIntervalMs = Long.parseLong(properties.get(PropertyAnalyzer
                     .PROPERTIES_GROUP_COMMIT_INTERVAL_MS));
             olapTable.readLock();

fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java

@@ -126,6 +126,15 @@ public class PropertyAnalyzer {
     public static final String PROPERTIES_INMEMORY = "in_memory";
 
     public static final String PROPERTIES_FILE_CACHE_TTL_SECONDS = "file_cache_ttl_seconds";
+    public static final long MIN_FILE_CACHE_TTL_SECONDS = 31536000L;
+    public static final String FILE_CACHE_TTL_CREATE_RESTRICTION_MSG =
+            PROPERTIES_FILE_CACHE_TTL_SECONDS + " temporarily only supports values greater than or equal to "
+                    + MIN_FILE_CACHE_TTL_SECONDS + " seconds (one year) to keep file cache resident. "
+                    + "The TTL feature will be restored in a future version.";
+    public static final String FILE_CACHE_TTL_ALTER_RESTRICTION_MSG =
+            "Modifying " + PROPERTIES_FILE_CACHE_TTL_SECONDS
+                    + " is not allowed in the current version. "
+                    + "The TTL feature will be available in a future version.";
 
     // _auto_bucket can only set in create table stmt rewrite bucket and can not be changed
     public static final String PROPERTIES_AUTO_BUCKET = "_auto_bucket";
@@ -322,7 +331,7 @@ public void rewrite(Map<String, String> properties) {
 
     public PropertyAnalyzer() {
         forceProperties = ImmutableList.of(
-                RewriteProperty.replace(PROPERTIES_FILE_CACHE_TTL_SECONDS, "0")
+                RewriteProperty.delete(PROPERTIES_FILE_CACHE_TTL_SECONDS)
                 );
     }
 
@@ -609,17 +618,23 @@ public static long analyzeTTL(Map<String, String> properties) throws AnalysisExc
             String ttlSecondsStr = properties.get(PROPERTIES_FILE_CACHE_TTL_SECONDS);
             try {
                 ttlSeconds = Long.parseLong(ttlSecondsStr);
-                if (ttlSeconds < 0) {
-                    throw new NumberFormatException();
-                }
             } catch (NumberFormatException e) {
-                throw new AnalysisException("The value " + ttlSecondsStr + " formats error or  is out of range "
-                           + "(0 < integer < Long.MAX_VALUE)");
+                throw new AnalysisException("The value " + ttlSecondsStr + " formats error or is out of range "
+                        + "(" + MIN_FILE_CACHE_TTL_SECONDS + " <= integer <= Long.MAX_VALUE)");
+            }
+            if (ttlSeconds < MIN_FILE_CACHE_TTL_SECONDS) {
+                throw new AnalysisException(FILE_CACHE_TTL_CREATE_RESTRICTION_MSG);
             }
         }
         return ttlSeconds;
     }
 
+    public static void analyzeTTLAlter(Map<String, String> properties) throws AnalysisException {
+        if (properties.containsKey(PROPERTIES_FILE_CACHE_TTL_SECONDS)) {
+            throw new AnalysisException(FILE_CACHE_TTL_ALTER_RESTRICTION_MSG);
+        }
+    }
+
     public static int analyzePartitionRetentionCount(Map<String, String> properties) throws AnalysisException {
         int retentionCount = -1;
         if (properties != null && properties.containsKey(PROPERTIES_PARTITION_RETENTION_COUNT)) {

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java

@@ -190,8 +190,7 @@ public CreateTableInfo(
         this.partitionTableInfo = partitionTableInfo;
         this.distribution = distribution;
         this.rollups = Utils.copyRequiredList(rollups);
-        this.properties = properties;
-        PropertyAnalyzer.getInstance().rewriteForceProperties(this.properties);
+        this.properties = analyzeTTLAndRewriteForceProperties(properties);
         this.extProperties = extProperties;
         this.clusterKeysColumnNames = Utils.copyRequiredList(clusterKeyColumnNames);
     }
@@ -234,8 +233,7 @@ public CreateTableInfo(
         this.partitionTableInfo = partitionTableInfo;
         this.distribution = distribution;
         this.rollups = Utils.copyRequiredList(rollups);
-        this.properties = properties;
-        PropertyAnalyzer.getInstance().rewriteForceProperties(this.properties);
+        this.properties = analyzeTTLAndRewriteForceProperties(properties);
         this.extProperties = extProperties;
         this.clusterKeysColumnNames = Utils.copyRequiredList(clusterKeyColumnNames);
     }
@@ -255,8 +253,17 @@ public CreateTableInfo(
         this.keys = keys;
         this.comment = comment;
         this.distribution = distribution;
-        this.properties = properties;
-        PropertyAnalyzer.getInstance().rewriteForceProperties(this.properties);
+        this.properties = analyzeTTLAndRewriteForceProperties(properties);
+    }
+
+    private static Map<String, String> analyzeTTLAndRewriteForceProperties(Map<String, String> properties) {
+        try {
+            PropertyAnalyzer.analyzeTTL(properties);
+        } catch (org.apache.doris.common.AnalysisException e) {
+            throw new AnalysisException(e.getMessage(), e.getCause());
+        }
+        PropertyAnalyzer.getInstance().rewriteForceProperties(properties);
+        return properties;
     }
 
     /**
@@ -488,6 +495,11 @@ public void validate(ConnectContext ctx) {
 
         if (engineName.equalsIgnoreCase(ENGINE_OLAP)) {
             boolean enableDuplicateWithoutKeysByDefault = false;
+            try {
+                PropertyAnalyzer.analyzeTTL(properties);
+            } catch (org.apache.doris.common.AnalysisException e) {
+                throw new AnalysisException(e.getMessage(), e.getCause());
+            }
             properties = PropertyAnalyzer.getInstance().rewriteOlapProperties(ctlName, dbName, properties);
 
             // In fuzzy tests, randomly set storage_format=V3 (ext_meta) for some tables.

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java

@@ -76,6 +76,8 @@ public void validate(ConnectContext ctx) throws UserException {
             throw new AnalysisException("Properties is not set");
         }
 
+        PropertyAnalyzer.analyzeTTLAlter(properties);
+
         if (properties.size() != 1
                 && !TableProperty.isSamePrefixProperties(
                         properties, DynamicPartitionProperty.DYNAMIC_PARTITION_PROPERTY_PREFIX)
@@ -341,9 +343,6 @@ public void validate(ConnectContext ctx) throws UserException {
             }
             this.needTableStable = false;
             this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
-        } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS)) {
-            this.needTableStable = false;
-            this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
         } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_PARTITION_RETENTION_COUNT)) {
             // do a check here for valid value
             int retentionCount = -1;

fe/fe-core/src/test/java/org/apache/doris/analysis/ModifyTablePropertiesTtlTest.java

@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.cloud.alter.CloudSchemaChangeHandler;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.nereids.trees.plans.commands.info.ModifyTablePropertiesOp;
+
+import com.google.common.collect.Maps;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Map;
+
+public class ModifyTablePropertiesTtlTest {
+    @Test
+    public void testLegacyAlterRejectsFileCacheTtl() {
+        ModifyTablePropertiesClause clause = new ModifyTablePropertiesClause(ttlProperties("31536000"));
+
+        AnalysisException exception = Assert.assertThrows(AnalysisException.class, clause::analyze);
+        assertTtlAlterRejected(exception);
+    }
+
+    @Test
+    public void testNereidsAlterRejectsFileCacheTtl() {
+        ModifyTablePropertiesOp op = new ModifyTablePropertiesOp(ttlProperties("31536000"));
+
+        AnalysisException exception = Assert.assertThrows(AnalysisException.class, () -> op.validate(null));
+        assertTtlAlterRejected(exception);
+    }
+
+    @Test
+    public void testMixedPropertiesStillRejectsFileCacheTTL() {
+        Map<String, String> properties = ttlProperties("31536000");
+        properties.put(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION, "true");
+
+        AnalysisException exception = Assert.assertThrows(AnalysisException.class,
+                () -> new ModifyTablePropertiesClause(properties).analyze());
+        assertTtlAlterRejected(exception);
+    }
+
+    @Test
+    public void testCloudSchemaChangeRejectsFileCacheTtlBeforeExecution() {
+        CloudSchemaChangeHandler handler = new CloudSchemaChangeHandler();
+
+        AnalysisException exception = Assert.assertThrows(AnalysisException.class,
+                () -> handler.updateTableProperties(null, "tbl", ttlProperties("31536000")));
+        assertTtlAlterRejected(exception);
+    }
+
+    private static Map<String, String> ttlProperties(String ttlSeconds) {
+        Map<String, String> properties = Maps.newHashMap();
+        properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS, ttlSeconds);
+        return properties;
+    }
+
+    private static void assertTtlAlterRejected(AnalysisException exception) {
+        Assert.assertTrue(exception.getMessage().contains(PropertyAnalyzer.FILE_CACHE_TTL_ALTER_RESTRICTION_MSG));
+    }
+}

fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java

@@ -24,6 +24,7 @@
 import org.apache.doris.common.ExceptionChecker;
 import org.apache.doris.common.FeConstants;
 import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.PropertyAnalyzer;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.resource.Tag;
 import org.apache.doris.utframe.TestWithFeService;
@@ -248,6 +249,34 @@ public void testNormal() throws DdlException, ConfigException {
                 + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1'); "));
     }
 
+    @Test
+    public void testFileCacheTtlCreateValidation() throws Exception {
+        ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
+                PropertyAnalyzer.FILE_CACHE_TTL_CREATE_RESTRICTION_MSG,
+                () -> createTable("create table test.tbl_ttl_low\n"
+                        + "(k1 int)\n"
+                        + "duplicate key(k1)\n"
+                        + "distributed by hash(k1) buckets 1\n"
+                        + "properties('replication_num' = '1', 'file_cache_ttl_seconds' = '31535999');"));
+
+        createTable("create table test.tbl_ttl_min\n"
+                + "(k1 int)\n"
+                + "duplicate key(k1)\n"
+                + "distributed by hash(k1) buckets 1\n"
+                + "properties('replication_num' = '1', 'file_cache_ttl_seconds' = '31536000');");
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("test");
+        OlapTable ttlTable = (OlapTable) db.getTableOrDdlException("tbl_ttl_min");
+        Assert.assertNotNull(ttlTable);
+
+        createTable("create table test.tbl_ttl_default\n"
+                + "(k1 int)\n"
+                + "duplicate key(k1)\n"
+                + "distributed by hash(k1) buckets 1\n"
+                + "properties('replication_num' = '1');");
+        OlapTable defaultTable = (OlapTable) db.getTableOrDdlException("tbl_ttl_default");
+        Assert.assertEquals(0L, defaultTable.getTTLSeconds());
+    }
+
     @Test
     public void testAbnormal() throws DdlException, ConfigException {
         ExceptionChecker.expectThrowsWithMsg(DdlException.class,

fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java

@@ -175,6 +175,41 @@ public void testStorageFormat() throws AnalysisException {
         PropertyAnalyzer.analyzeStorageFormat(propertiesV1);
     }
 
+    @Test
+    public void testAnalyzeTtl() throws AnalysisException {
+        Map<String, String> properties = Maps.newHashMap();
+        Assert.assertEquals(0L, PropertyAnalyzer.analyzeTTL(properties));
+
+        properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
+                String.valueOf(PropertyAnalyzer.MIN_FILE_CACHE_TTL_SECONDS));
+        Assert.assertEquals(PropertyAnalyzer.MIN_FILE_CACHE_TTL_SECONDS, PropertyAnalyzer.analyzeTTL(properties));
+
+        properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS, String.valueOf(Long.MAX_VALUE));
+        Assert.assertEquals(Long.MAX_VALUE, PropertyAnalyzer.analyzeTTL(properties));
+
+        properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS,
+                String.valueOf(PropertyAnalyzer.MIN_FILE_CACHE_TTL_SECONDS - 1));
+        AnalysisException exception = Assert.assertThrows(AnalysisException.class,
+                () -> PropertyAnalyzer.analyzeTTL(properties));
+        Assert.assertTrue(exception.getMessage().contains(PropertyAnalyzer.FILE_CACHE_TTL_CREATE_RESTRICTION_MSG));
+
+        properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS, "0");
+        exception = Assert.assertThrows(AnalysisException.class, () -> PropertyAnalyzer.analyzeTTL(properties));
+        Assert.assertTrue(exception.getMessage().contains(PropertyAnalyzer.FILE_CACHE_TTL_CREATE_RESTRICTION_MSG));
+
+        properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS, "-1");
+        exception = Assert.assertThrows(AnalysisException.class, () -> PropertyAnalyzer.analyzeTTL(properties));
+        Assert.assertTrue(exception.getMessage().contains(PropertyAnalyzer.FILE_CACHE_TTL_CREATE_RESTRICTION_MSG));
+
+        properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS, "invalid");
+        exception = Assert.assertThrows(AnalysisException.class, () -> PropertyAnalyzer.analyzeTTL(properties));
+        Assert.assertTrue(exception.getMessage().contains("formats error or is out of range"));
+
+        properties.put(PropertyAnalyzer.PROPERTIES_FILE_CACHE_TTL_SECONDS, "9223372036854775808");
+        exception = Assert.assertThrows(AnalysisException.class, () -> PropertyAnalyzer.analyzeTTL(properties));
+        Assert.assertTrue(exception.getMessage().contains("formats error or is out of range"));
+    }
+
     @Test
     public void testTag() throws AnalysisException {
         HashMap<String, String> properties = Maps.newHashMap();