Flatten logical expressions (#801)

* Add logical flatten optimization rule

* Fix serializer for NOT

* Not qualify the identifier for simple SQL generated

* Fix optimization

* Fix UT

component/component-commons/src/main/java/org/bithon/component/commons/expression/LogicalExpression.java

@@ -161,6 +161,8 @@ public OR(List<IExpression> operands) {
         }
 
         public OR(IExpression... operands) {
+            // Arrays.asList returns unmodified copy,
+            // but we need a modifiable one so that further optimizer can be applied on this expression
             super(OR, new ArrayList<>(Arrays.asList(operands)));
         }
 
@@ -187,6 +189,12 @@ public NOT(List<IExpression> operands) {
             super(NOT, operands);
         }
 
+        public NOT(IExpression... operands) {
+            // Arrays.asList returns unmodified copy,
+            // but we need a modifiable one so that further optimizer can be applied on this expression
+            super(NOT, new ArrayList<>(Arrays.asList(operands)));
+        }
+
         public NOT(IExpression expression) {
             super(NOT, new ArrayList<>(Collections.singletonList(expression)));
         }

component/component-commons/src/main/java/org/bithon/component/commons/expression/optimzer/ExpressionOptimizer.java

@@ -32,6 +32,7 @@
 import org.bithon.component.commons.expression.MapAccessExpression;
 
 import java.util.Iterator;
+import java.util.List;
 
 /**
  * @author Frank Chen
@@ -52,7 +53,32 @@ public IExpression visit(LiteralExpression expression) {
 
         @Override
         public IExpression visit(LogicalExpression expression) {
-            expression.getOperands().replaceAll(iExpression -> iExpression.accept(this));
+            List<IExpression> operands = expression.getOperands();
+
+            for (int i = 0; i < operands.size(); i++) {
+                // Apply optimization on the operand
+                IExpression expr = operands.get(i).accept(this);
+
+                if (expression instanceof LogicalExpression.AND && expr instanceof LogicalExpression.AND
+                    || (expression instanceof LogicalExpression.OR && expr instanceof LogicalExpression.OR)
+                    || (expression instanceof LogicalExpression.NOT && expr instanceof LogicalExpression.AND)
+                ) {
+                    operands.remove(i);
+
+                    // Flatten nested AND/OR expression
+                    List<IExpression> nestedExpressions = ((LogicalExpression) expr).getOperands();
+                    for (IExpression nest : nestedExpressions) {
+                        operands.add(i++, nest);
+                    }
+
+                    // The nested has N elements, since we remove one element first,
+                    // the number total added elements is N - 1
+                    i--;
+                } else {
+                    operands.set(i, expr);
+                }
+            }
+
             return expression;
         }
 
@@ -177,9 +203,11 @@ public IExpression visit(ConditionalExpression expression) {
     }
 
     /**
-     * Simplifies constant expressions in logical AND/OR/NOT.
+     * 1. Simplifies constant expressions in logical AND/OR/NOT.
      * For example, the expression '1 = 1 AND condition2' can be simplified as condition2.
      * '1 = 1 OR condition2' can be simplified as true.
+     * 2. Reverse the logical expressions.
+     * For example, NOT a = 1 will be optimized into a != 1
      */
     static class LogicalExpressionOptimizer extends AbstractOptimizer {
         @Override
@@ -260,11 +288,11 @@ private IExpression handleNotExpression(LogicalExpression expression) {
             } else if (subExpression instanceof ComparisonExpression.LTE) {
                 // Turn '<= into '>'
                 return new ComparisonExpression.GT(((ComparisonExpression.LTE) subExpression).getLeft(),
-                                                  ((ComparisonExpression.LTE) subExpression).getRight());
+                                                   ((ComparisonExpression.LTE) subExpression).getRight());
             } else if (subExpression instanceof ComparisonExpression.GTE) {
                 // Turn '>= into '<'
                 return new ComparisonExpression.LT(((ComparisonExpression.GTE) subExpression).getLeft(),
-                                                  ((ComparisonExpression.GTE) subExpression).getRight());
+                                                   ((ComparisonExpression.GTE) subExpression).getRight());
             }
 
             return expression;

component/component-commons/src/main/java/org/bithon/component/commons/expression/serialization/ExpressionSerializer.java

@@ -77,18 +77,17 @@ public boolean visit(LiteralExpression expression) {
 
     @Override
     public boolean visit(LogicalExpression expression) {
+        String concatOperator = expression.getOperator();
         if (expression instanceof LogicalExpression.NOT) {
-            sb.append("NOT (");
-            expression.getOperands().get(0).accept(this);
-            sb.append(")");
-            return false;
+            sb.append("NOT ");
+            concatOperator = "AND";
         }
 
         sb.append('(');
         for (int i = 0, size = expression.getOperands().size(); i < size; i++) {
             if (i > 0) {
                 sb.append(' ');
-                sb.append(expression.getOperator());
+                sb.append(concatOperator);
                 sb.append(' ');
             }
             expression.getOperands().get(i).accept(this);

component/component-commons/src/test/java/org/bithon/component/commons/expression/optimizer/ExpressionOptimizerTest.java

@@ -0,0 +1,139 @@
+/*
+ *    Copyright 2020 bithon.org
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package org.bithon.component.commons.expression.optimizer;
+
+import org.bithon.component.commons.expression.ComparisonExpression;
+import org.bithon.component.commons.expression.IdentifierExpression;
+import org.bithon.component.commons.expression.LiteralExpression;
+import org.bithon.component.commons.expression.LogicalExpression;
+import org.bithon.component.commons.expression.optimzer.ExpressionOptimizer;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * NOTE:
+ * Most tests are placed in the server-storage module because it provides AST parser for simpler test construction
+ *
+ * @author frank.chen021@outlook.com
+ * @date 5/6/24 10:15 am
+ */
+public class ExpressionOptimizerTest {
+
+    @Test
+    public void testLogicalExpression_FlattenAND() {
+        LogicalExpression expr = new LogicalExpression.AND(
+            new ComparisonExpression.EQ(new IdentifierExpression("a"), LiteralExpression.create(1)),
+
+            new LogicalExpression.AND(
+                new ComparisonExpression.EQ(new IdentifierExpression("b"), LiteralExpression.create(2)),
+                new ComparisonExpression.EQ(new IdentifierExpression("c"), LiteralExpression.create(3))
+            ),
+
+            new LogicalExpression.AND(
+                new ComparisonExpression.EQ(new IdentifierExpression("d"), LiteralExpression.create(4)),
+                new ComparisonExpression.EQ(new IdentifierExpression("e"), LiteralExpression.create(5))
+            )
+        );
+
+        expr.accept(new ExpressionOptimizer.AbstractOptimizer());
+
+        Assert.assertEquals(5, expr.getOperands().size());
+        Assert.assertEquals("(a = 1 AND b = 2 AND c = 3 AND d = 4 AND e = 5)", expr.serializeToText(null));
+    }
+
+    @Test
+    public void testLogicalExpression_FlattenOR() {
+        LogicalExpression expr = new LogicalExpression.OR(
+            new ComparisonExpression.EQ(new IdentifierExpression("a"), LiteralExpression.create(1)),
+
+            new LogicalExpression.OR(
+                new ComparisonExpression.EQ(new IdentifierExpression("b"), LiteralExpression.create(2)),
+                new ComparisonExpression.EQ(new IdentifierExpression("c"), LiteralExpression.create(3))
+            )
+        );
+
+        expr.accept(new ExpressionOptimizer.AbstractOptimizer());
+
+        Assert.assertEquals(3, expr.getOperands().size());
+        Assert.assertEquals("(a = 1 OR b = 2 OR c = 3)", expr.serializeToText(null));
+    }
+
+    @Test
+    public void testLogicalExpression_NoFlatten() {
+        LogicalExpression expr = new LogicalExpression.AND(
+            new ComparisonExpression.EQ(new IdentifierExpression("a"), LiteralExpression.create(1)),
+
+            new LogicalExpression.OR(
+                new ComparisonExpression.EQ(new IdentifierExpression("b"), LiteralExpression.create(2)),
+                new ComparisonExpression.EQ(new IdentifierExpression("c"), LiteralExpression.create(3))
+            )
+        );
+
+        expr.accept(new ExpressionOptimizer.AbstractOptimizer());
+
+        Assert.assertEquals(2, expr.getOperands().size());
+        Assert.assertEquals("(a = 1 AND (b = 2 OR c = 3))", expr.serializeToText(null));
+    }
+
+    @Test
+    public void testLogicalExpression_FlattenRecursively() {
+        LogicalExpression expr = new LogicalExpression.AND(
+            new ComparisonExpression.EQ(new IdentifierExpression("a"), LiteralExpression.create(1)),
+
+            new LogicalExpression.AND(
+                new ComparisonExpression.EQ(new IdentifierExpression("b"), LiteralExpression.create(2)),
+
+                new LogicalExpression.AND(
+                    new ComparisonExpression.EQ(new IdentifierExpression("c"), LiteralExpression.create(3)),
+
+                    new LogicalExpression.AND(
+                        new ComparisonExpression.EQ(new IdentifierExpression("d"), LiteralExpression.create(4))
+                        )
+                )
+            )
+        );
+
+        expr.accept(new ExpressionOptimizer.AbstractOptimizer());
+
+        Assert.assertEquals(4, expr.getOperands().size());
+        Assert.assertEquals("(a = 1 AND b = 2 AND c = 3 AND d = 4)", expr.serializeToText(null));
+    }
+
+    @Test
+    public void testLogicalExpression_FlattenNOT() {
+        LogicalExpression expr = new LogicalExpression.NOT(
+            new ComparisonExpression.EQ(new IdentifierExpression("a"), LiteralExpression.create(1)),
+
+            new LogicalExpression.AND(
+                new ComparisonExpression.EQ(new IdentifierExpression("b"), LiteralExpression.create(2)),
+
+                new LogicalExpression.AND(
+                    new ComparisonExpression.EQ(new IdentifierExpression("c"), LiteralExpression.create(3)),
+
+                    new LogicalExpression.AND(
+                        new ComparisonExpression.EQ(new IdentifierExpression("d"), LiteralExpression.create(4))
+                    )
+                )
+            )
+        );
+
+        expr.accept(new ExpressionOptimizer.AbstractOptimizer());
+
+        Assert.assertEquals(4, expr.getOperands().size());
+        Assert.assertEquals("NOT (a = 1 AND b = 2 AND c = 3 AND d = 4)", expr.serializeToText(null));
+    }
+}

server/storage-jdbc-clickhouse/src/test/java/org/bithon/server/storage/jdbc/clickhouse/common/optimizer/HasTokenFunctionOptimizerTest.java

@@ -75,6 +75,6 @@ public void test_Replaced_In_CompoundExpression() {
                                                .build("hasToken(a, 'SERVER_ERROR') AND hasToken(a, 'EXCEPTION_CODE')")
                                                .accept(new HasTokenFunctionOptimizer());
 
-        Assert.assertEquals("((hasToken(a, 'SERVER') AND hasToken(a, 'ERROR') AND a like '%SERVER_ERROR%') AND (hasToken(a, 'EXCEPTION') AND hasToken(a, 'CODE') AND a like '%EXCEPTION_CODE%'))", expr.serializeToText(null));
+        Assert.assertEquals("(hasToken(a, 'SERVER') AND hasToken(a, 'ERROR') AND a like '%SERVER_ERROR%' AND hasToken(a, 'EXCEPTION') AND hasToken(a, 'CODE') AND a like '%EXCEPTION_CODE%')", expr.serializeToText(null));
     }
 }

server/storage-jdbc/src/main/java/org/bithon/server/storage/jdbc/common/dialect/Expression2Sql.java

@@ -29,18 +29,18 @@
 public class Expression2Sql extends ExpressionSerializer {
 
     public static String from(ISchema dataSource, ISqlDialect sqlDialect, IExpression expression) {
-        return from(dataSource.getDataStoreSpec().getStore(), sqlDialect, expression);
+        return from((String) null, sqlDialect, expression);
     }
 
     public static String from(String qualifier, ISqlDialect sqlDialect, IExpression expression) {
         if (expression == null) {
             return null;
         }
-        return new Expression2Sql(qualifier, sqlDialect).serialize(sqlDialect.transform(expression));
-    }
 
-    public static String from(ISqlDialect sqlDialect, IExpression expression) {
-        return new Expression2Sql(null, sqlDialect).serialize(sqlDialect.transform(expression));
+        // Apply DB-related transformation on general AST
+        IExpression transformed = sqlDialect.transform(expression);
+
+        return new Expression2Sql(qualifier, sqlDialect).serialize(transformed);
     }
 
     private final ISqlDialect sqlDialect;