branch-4.1: [fix](fe) Preserve narrowing datetimev2 casts in simplify in predicate #63343

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java

@@ -63,10 +63,12 @@ && canLosslessConvertToDateV2Literal((DateTimeV2Literal) literal))) {
                 } else if (cast.child().getDataType().isDateTimeV2Type()
                         && expr.child(1) instanceof DateTimeV2Literal) {
                     List<Expression> literals = expr.children().subList(1, expr.children().size());
+                    DateTimeV2Type castType = (DateTimeV2Type) cast.getDataType();
                     DateTimeV2Type compareType = (DateTimeV2Type) cast.child().getDataType();
-                    if (literals.stream().allMatch(literal -> literal instanceof DateTimeV2Literal
-                            && canLosslessConvertToLowScaleLiteral(
-                            (DateTimeV2Literal) literal, compareType.getScale()))) {
+                    if (castType.getScale() >= compareType.getScale()
+                            && literals.stream().allMatch(literal -> literal instanceof DateTimeV2Literal
+                                    && canLosslessConvertToLowScaleLiteral(
+                                            (DateTimeV2Literal) literal, compareType.getScale()))) {
                         ImmutableList.Builder<Expression> children = ImmutableList.builder();
                         children.add(cast.child());
                         literals.forEach(l -> children.add(new DateTimeV2Literal(compareType,
@@ -99,6 +101,7 @@ private static DateV2Literal convertToDateV2Literal(DateTimeV2Literal literal) {
     }
 
     private static boolean canLosslessConvertToLowScaleLiteral(DateTimeV2Literal literal, int targetScale) {
-        return literal.getMicroSecond() % (1L << (DateTimeV2Type.MAX_SCALE - targetScale)) == 0;
+        long scaleFactor = (long) Math.pow(10, DateTimeV2Type.MAX_SCALE - targetScale);
+        return literal.getMicroSecond() % scaleFactor == 0;
     }
 }

fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyInPredicateTest.java

@@ -31,14 +31,22 @@
 
 public class SimplifyInPredicateTest extends ExpressionRewriteTestHelper {
 
+    private ExpressionRuleExecutor newRewriteExecutor() {
+        return new ExpressionRuleExecutor(ImmutableList.of(
+                bottomUp(
+                        FoldConstantRule.INSTANCE,
+                        SimplifyInPredicate.INSTANCE)));
+    }
+
+    private ExpressionRuleExecutor newFoldExecutor() {
+        return new ExpressionRuleExecutor(ImmutableList.of(
+                bottomUp(
+                        FoldConstantRule.INSTANCE)));
+    }
+
     @Test
     public void test() {
-        executor = new ExpressionRuleExecutor(ImmutableList.of(
-                bottomUp(
-                    FoldConstantRule.INSTANCE,
-                    SimplifyInPredicate.INSTANCE
-                )
-        ));
+        executor = newRewriteExecutor();
         Map<String, Slot> mem = Maps.newHashMap();
         Expression rewrittenExpression = PARSER.parseExpression("cast(CA as DATETIME) in ('1992-01-31 00:00:00', '1992-02-01 00:00:00')");
         // after parse and type coercion: CAST(CAST(CA AS DATETIMEV2(0)) AS DATETIMEV2(6)) IN ('1992-01-31 00:00:00.000000', '1992-02-01 00:00:00.000000')
@@ -49,12 +57,44 @@ public void test() {
         rewrittenExpression = executor.rewrite(rewrittenExpression, context);
         Expression expectedExpression = PARSER.parseExpression("CA in (cast('1992-01-31' as date), cast('1992-02-01' as date))");
         expectedExpression = replaceUnboundSlot(expectedExpression, mem);
-        executor = new ExpressionRuleExecutor(ImmutableList.of(
-            bottomUp(
-                FoldConstantRule.INSTANCE
-            )
-        ));
+        executor = newFoldExecutor();
         expectedExpression = executor.rewrite(expectedExpression, context);
         Assertions.assertEquals(expectedExpression, rewrittenExpression);
     }
+
+    @Test
+    public void testDoNotEliminateNarrowingDateTimeV2Cast() {
+        ExpressionRuleExecutor rewriteExecutor = newRewriteExecutor();
+        ExpressionRuleExecutor foldExecutor = newFoldExecutor();
+        Map<String, Slot> mem = Maps.newHashMap();
+        Expression rewrittenExpression = PARSER.parseExpression("cast(AA as DATETIMEV2(3)) in "
+                + "('2024-01-01 12:34:56.123000', '2024-01-01 09:30:01.000000', '2024-01-01 22:00:00.000000')");
+        rewrittenExpression = typeCoercion(replaceUnboundSlot(rewrittenExpression, mem));
+        rewrittenExpression = rewriteExecutor.rewrite(rewrittenExpression, context);
+        Expression rewrittenAgain = rewriteExecutor.rewrite(rewrittenExpression, context);
+
+        Expression expectedExpression = PARSER.parseExpression("cast(AA as DATETIMEV2(3)) in "
+                + "(cast('2024-01-01 12:34:56.123' as DATETIMEV2(3)), "
+                + "cast('2024-01-01 09:30:01.000' as DATETIMEV2(3)), "
+                + "cast('2024-01-01 22:00:00.000' as DATETIMEV2(3)))");
+        expectedExpression = replaceUnboundSlot(expectedExpression, mem);
+        expectedExpression = foldExecutor.rewrite(expectedExpression, context);
+
+        Assertions.assertEquals(expectedExpression, rewrittenExpression);
+        Assertions.assertEquals(rewrittenExpression, rewrittenAgain);
+    }
+
+    @Test
+    public void testDateTimeV2LiteralMustAlignWithTargetScale() {
+        ExpressionRuleExecutor rewriteExecutor = newRewriteExecutor();
+        Map<String, Slot> mem = Maps.newHashMap();
+        Expression rewrittenExpression = PARSER.parseExpression("cast(cast(AA as DATETIMEV2(3)) as DATETIMEV2(6)) "
+                + "in ('2024-01-01 12:34:56.123128')");
+        rewrittenExpression = typeCoercion(replaceUnboundSlot(rewrittenExpression, mem));
+        Expression originalExpression = rewrittenExpression;
+
+        rewrittenExpression = rewriteExecutor.rewrite(rewrittenExpression, context);
+
+        Assertions.assertEquals(originalExpression, rewrittenExpression);
+    }
 }

regression-test/data/nereids_syntax_p0/test_simplify_in_predicate.out

@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !datetimev2_narrow_cast --
+1
+2
+3
+4
+

regression-test/suites/nereids_syntax_p0/test_simplify_in_predicate.groovy

@@ -35,4 +35,32 @@ suite("test_simplify_in_predicate") {
         sql "verbose select * from test_simplify_in_predicate_t where a in ('1992-01-31', '1992-02-01', '1992-02-02', '1992-02-03', '1992-02-04');"
         notContains "CAST"
     }
+
+    sql 'drop table if exists test_simplify_in_predicate_datetimev2_t'
+    sql """CREATE TABLE IF NOT EXISTS `test_simplify_in_predicate_datetimev2_t` (
+            id INT NOT NULL,
+            ts6 DATETIMEV2(6) NOT NULL
+            ) ENGINE=OLAP
+            DUPLICATE KEY(`id`)
+            DISTRIBUTED BY HASH(`id`) BUCKETS 1
+            PROPERTIES (
+            "replication_num" = "1"
+            );"""
+    sql """INSERT INTO test_simplify_in_predicate_datetimev2_t VALUES
+            (1, '2024-01-01 12:34:56.123456'),
+            (2, '2024-01-01 09:30:00.999999'),
+            (3, '2024-01-01 09:30:01.000000'),
+            (4, '2024-01-01 22:00:00.000001');"""
+
+    order_qt_datetimev2_narrow_cast """
+            SELECT id
+            FROM test_simplify_in_predicate_datetimev2_t
+            WHERE CAST(ts6 AS DATETIMEV2(3)) IN (
+                CAST('2024-01-01 12:34:56.123000' AS DATETIMEV2(6)),
+                CAST('2024-01-01 09:30:01.000000' AS DATETIMEV2(6)),
+                CAST('2024-01-01 22:00:00.000000' AS DATETIMEV2(6))
+            )
+            ORDER BY 1
+            """
+
 }