DRILL-6870: Upgrade to ANTLR4

closes #1554

common/pom.xml

@@ -101,12 +101,6 @@
       <artifactId>metrics-jmx</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>org.antlr</groupId>
-      <artifactId>antlr-runtime</artifactId>
-      <version>3.4</version>
-    </dependency>
-
     <dependency>
         <groupId>joda-time</groupId>
         <artifactId>joda-time</artifactId>

contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java

@@ -18,22 +18,19 @@
 
 package org.apache.drill.exec.planner.index;
 
+import org.apache.drill.common.parser.LogicalExpressionParser;
 import com.mapr.db.Admin;
 import com.mapr.db.MapRDB;
 import com.mapr.db.exceptions.DBException;
 import com.mapr.db.index.IndexDesc;
 import com.mapr.db.index.IndexDesc.MissingAndNullOrdering;
 import com.mapr.db.index.IndexFieldDesc;
-import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.CommonTokenStream;
 import org.apache.calcite.rel.RelFieldCollation;
 import org.apache.calcite.rel.RelFieldCollation.NullDirection;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.parser.ExprLexer;
-import org.apache.drill.common.expression.parser.ExprParser;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.common.util.DrillFileUtils;
@@ -67,8 +64,7 @@
 
 public class MapRDBIndexDiscover extends IndexDiscoverBase implements IndexDiscover {
 
-  static final String DEFAULT_STRING_CAST_LEN_STR = "256";
-  static final String FIELD_DELIMITER = ":";
+  public static final String DEFAULT_STRING_CAST_LEN_STR = "256";
 
   public MapRDBIndexDiscover(GroupScan inScan, DrillScanRelBase scanRel) {
     super((AbstractDbGroupScan) inScan, scanRel);
@@ -86,7 +82,7 @@ public IndexCollection getTableIndex(String tableName) {
   /**
    * For a given table name get the list of indexes defined on the table according to the visibility of
    * the indexes based on permissions.
-   * @param tableName
+   * @param tableName table name
    * @return an IndexCollection representing the list of indexes for that table
    */
   private IndexCollection getTableIndexFromMFS(String tableName) {
@@ -137,7 +133,7 @@ public DrillTable getNativeDrillTable(IndexDescriptor idxDescriptor) {
         return null;
       }
       MapRDBFormatPlugin maprFormatPlugin = ((MapRDBGroupScan) origScan).getFormatPlugin();
-      FileSystemPlugin fsPlugin = (FileSystemPlugin) (((MapRDBGroupScan) origScan).getStoragePlugin());
+      FileSystemPlugin fsPlugin = (FileSystemPlugin) (origScan.getStoragePlugin());
 
       DrillFileSystem fs = ImpersonationUtil.createFileSystem(origScan.getUserName(), fsPlugin.getFsConf());
       MapRDBFormatMatcher matcher = (MapRDBFormatMatcher) (maprFormatPlugin.getMatcher());
@@ -240,14 +236,9 @@ private LogicalExpression castFunctionSQLSyntax(String field, String type) throw
     }
     try {
       String castFunc = String.format("cast( %s as %s)", field, castTypeStr);
-      final ExprLexer lexer = new ExprLexer(new ANTLRStringStream(castFunc));
-      final CommonTokenStream tokens = new CommonTokenStream(lexer);
-      final ExprParser parser = new ExprParser(tokens);
-      final ExprParser.parse_return ret = parser.parse();
-      logger.trace("{}, {}", tokens, ret);
-      return ret.e;
-    }catch(Exception ex) {
-      logger.error("parse failed{}", ex);
+      return LogicalExpressionParser.parse(castFunc);
+    } catch (Exception ex) {
+      logger.error("parse failed: {}", ex);
     }
     return null;
   }
@@ -313,8 +304,7 @@ private CollationContext buildCollationContext(List<LogicalExpression> indexFiel
     for (int i = 0; i < indexFieldCollations.size(); i++) {
       collationMap.put(indexFields.get(i), indexFieldCollations.get(i));
     }
-    CollationContext collationContext = new CollationContext(collationMap, indexFieldCollations);
-    return collationContext;
+    return new CollationContext(collationMap, indexFieldCollations);
   }
 
   private DrillIndexDescriptor buildIndexDescriptor(String tableName, IndexDesc desc)
@@ -363,11 +353,7 @@ private Admin admin() {
 
     final Admin admin;
     try {
-      admin = currentUser.doAs(new PrivilegedExceptionAction<Admin>() {
-        public Admin run() throws Exception {
-          return MapRDB.getAdmin(conf);
-        }
-      });
+      admin = currentUser.doAs((PrivilegedExceptionAction<Admin>) () -> MapRDB.getAdmin(conf));
     } catch (Exception e) {
       throw new DrillRuntimeException("Failed to get Admin instance for user: " + currentUser.getUserName(), e);
     }

exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java

@@ -18,16 +18,11 @@
 package org.apache.drill.exec;
 
 import com.codahale.metrics.MetricRegistry;
-import org.apache.drill.shaded.guava.com.google.common.io.Files;
+import org.apache.drill.common.parser.LogicalExpressionParser;
 import mockit.Mock;
 import mockit.MockUp;
-import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.CommonTokenStream;
-import org.antlr.runtime.RecognitionException;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.parser.ExprLexer;
-import org.apache.drill.common.expression.parser.ExprParser;
 import org.apache.drill.common.scanner.ClassPathScanner;
 import org.apache.drill.exec.compile.CodeCompilerTestFactory;
 import org.apache.drill.exec.memory.RootAllocatorFactory;
@@ -48,7 +43,6 @@
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
 
-import java.io.File;
 import java.io.IOException;
 import java.text.DateFormatSymbols;
 import java.util.Locale;
@@ -94,17 +88,6 @@ public static FileSystem getLocalFileSystem() throws IOException {
     return FileSystem.get(configuration);
   }
 
-  /**
-   * Create a temp directory to store the given <i>dirName</i>.
-   * Directory will be deleted on exit.
-   * @param dirName directory name
-   * @return Full path including temp parent directory and given directory name.
-   */
-  public static String getTempDir(final String dirName) {
-    final File dir = Files.createTempDir();
-    return dir.getAbsolutePath() + File.separator + dirName;
-  }
-
   protected DrillbitContext mockDrillbitContext() throws Exception {
     final DrillbitContext context = mock(DrillbitContext.class);
 
@@ -118,12 +101,8 @@ protected DrillbitContext mockDrillbitContext() throws Exception {
     return context;
   }
 
-  protected LogicalExpression parseExpr(String expr) throws RecognitionException {
-    final ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
-    final CommonTokenStream tokens = new CommonTokenStream(lexer);
-    final ExprParser parser = new ExprParser(tokens);
-    final ExprParser.parse_return ret = parser.parse();
-    return ret.e;
+  public LogicalExpression parseExpr(String expr) {
+    return LogicalExpressionParser.parse(expr);
   }
 
   /**

exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java

@@ -17,68 +17,46 @@
  */
 package org.apache.drill.exec.compile;
 
-import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.CommonTokenStream;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.parser.ExprLexer;
-import org.apache.drill.common.expression.parser.ExprParser;
-import org.apache.drill.common.expression.parser.ExprParser.parse_return;
+import org.apache.drill.common.parser.LogicalExpressionParser;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.expr.CodeGenerator;
 import org.apache.drill.exec.expr.EvaluationVisitor;
 import org.apache.drill.exec.expr.ValueVectorReadExpression;
 import org.apache.drill.exec.expr.ValueVectorWriteExpression;
-import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
 import org.apache.drill.exec.physical.impl.project.Projector;
 import org.apache.drill.exec.record.TypedFieldId;
 import org.junit.Test;
 
 public class TestEvaluationVisitor {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestEvaluationVisitor.class);
-
 
   @Test
-  public void x() throws Exception{
-    DrillConfig c = DrillConfig.create();
-
-    FunctionImplementationRegistry reg = new FunctionImplementationRegistry(c);
+  public void testEvaluation() {
     EvaluationVisitor v = new EvaluationVisitor();
     CodeGenerator<?> g = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, null);
-    SchemaPath path = (SchemaPath) getExpr("a.b[4][2].c[6]");
+    SchemaPath path = (SchemaPath) LogicalExpressionParser.parse("a.b[4][2].c[6]");
 
-    TypedFieldId id = TypedFieldId.newBuilder() //
-      .addId(1) //
-      .addId(3) //
-      .remainder(path.getRootSegment()) //
+    TypedFieldId id = TypedFieldId.newBuilder()
+      .addId(1)
+      .addId(3)
+      .remainder(path.getRootSegment())
       .intermediateType(Types.optional(MinorType.MAP))
-      .finalType(Types.repeated(MinorType.MAP)) //
-      .hyper() //
-      .withIndex() //
+      .finalType(Types.repeated(MinorType.MAP))
+      .hyper()
+      .withIndex()
       .build();
 
     ValueVectorReadExpression e = new ValueVectorReadExpression(id);
 
-    TypedFieldId outId = TypedFieldId.newBuilder() //
-        .addId(1) //
-        .finalType(Types.repeated(MinorType.MAP)) //
-        .intermediateType(Types.repeated(MinorType.MAP)) //
+    TypedFieldId outId = TypedFieldId.newBuilder()
+        .addId(1)
+        .finalType(Types.repeated(MinorType.MAP))
+        .intermediateType(Types.repeated(MinorType.MAP))
         .build();
     ValueVectorWriteExpression e2 = new ValueVectorWriteExpression(outId, e, true);
 
     v.addExpr(e2,  g.getRoot());
-    logger.debug(g.generateAndGet());
   }
 
-  private LogicalExpression getExpr(String expr) throws Exception{
-    ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
-    CommonTokenStream tokens = new CommonTokenStream(lexer);
-    ExprParser parser = new ExprParser(tokens);
-    parse_return ret = parser.parse();
-
-    return ret.e;
-
-  }
 }

exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/unnest/TestUnnestWithLateralCorrectness.java

@@ -37,7 +37,7 @@
 import org.apache.drill.exec.physical.impl.sort.RecordBatchData;
 import org.apache.drill.exec.planner.common.DrillLateralJoinRelBase;
 import org.apache.drill.exec.planner.common.DrillUnnestRelBase;
-import org.apache.drill.exec.planner.logical.DrillLogicalTestutils;
+import org.apache.drill.exec.planner.logical.DrillLogicalTestUtils;
 import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.record.VectorContainer;
 import org.apache.drill.exec.record.metadata.TupleMetadata;
@@ -590,7 +590,7 @@ private <T> void testUnnest( TupleMetadata[] incomingSchemas,
 
     // project is required to rename the columns so as to disambiguate the same column name from
     // unnest operator and the regular scan.
-    final Project projectPopConfig = new Project(DrillLogicalTestutils.parseExprs("unnestColumn", "unnestColumn1",
+    final Project projectPopConfig = new Project(DrillLogicalTestUtils.parseExprs("unnestColumn", "unnestColumn1",
       unnestPopConfig.getImplicitColumn(), unnestPopConfig.getImplicitColumn()), null);
 
     final ProjectRecordBatch projectBatch =
@@ -899,12 +899,12 @@ private <T> void testNestedUnnest( TupleMetadata[] incomingSchemas,
 
     // Create intermediate Project
     final Project projectPopConfig1 =
-        new Project(DrillLogicalTestutils.parseExprs("unnestColumn.colB", "colB",
+        new Project(DrillLogicalTestUtils.parseExprs("unnestColumn.colB", "colB",
           unnestPopConfig1.getImplicitColumn(), unnestPopConfig1.getImplicitColumn()), unnestPopConfig1);
     final ProjectRecordBatch projectBatch1 =
         new ProjectRecordBatch(projectPopConfig1, unnestBatch1, fixture.getFragmentContext());
     final Project projectPopConfig2 =
-        new Project(DrillLogicalTestutils.parseExprs("colB", "unnestColumn2",
+        new Project(DrillLogicalTestUtils.parseExprs("colB", "unnestColumn2",
           unnestPopConfig2.getImplicitColumn(), unnestPopConfig2.getImplicitColumn()), unnestPopConfig2);
     final ProjectRecordBatch projectBatch2 =
         new ProjectRecordBatch(projectPopConfig2, unnestBatch2, fixture.getFragmentContext());

exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/DrillLogicalTestutils.java → exec/java-exec/src/test/java/org/apache/drill/exec/planner/logical/DrillLogicalTestUtils.java

@@ -17,54 +17,40 @@
  */
 package org.apache.drill.exec.planner.logical;
 
+import org.apache.drill.common.parser.LogicalExpressionParser;
 import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
-import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.CommonTokenStream;
-import org.antlr.runtime.RecognitionException;
 import org.apache.calcite.rel.RelFieldCollation;
 import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.PathSegment;
 import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.parser.ExprLexer;
-import org.apache.drill.common.expression.parser.ExprParser;
 import org.apache.drill.common.logical.data.JoinCondition;
 import org.apache.drill.common.logical.data.NamedExpression;
 import org.apache.drill.common.logical.data.Order;
 
 import java.util.ArrayList;
 import java.util.List;
 
-public class DrillLogicalTestutils {
+public class DrillLogicalTestUtils {
   public static Order.Ordering ordering(String expression,
                                         RelFieldCollation.Direction direction,
                                         RelFieldCollation.NullDirection nullDirection) {
-    return new Order.Ordering(direction, parseExpr(expression), nullDirection);
+    return new Order.Ordering(direction, LogicalExpressionParser.parse(expression), nullDirection);
   }
 
   public static JoinCondition joinCond(String leftExpr, String relationship, String rightExpr) {
-    return new JoinCondition(relationship, parseExpr(leftExpr), parseExpr(rightExpr));
+    return new JoinCondition(relationship, LogicalExpressionParser.parse(leftExpr), LogicalExpressionParser.parse(rightExpr));
   }
 
   public static List<NamedExpression> parseExprs(String... expressionsAndOutputNames) {
     Preconditions.checkArgument(expressionsAndOutputNames.length % 2 == 0,
-      "List of expressions and output field names" + " is not complete, each expression must explicitly give and output name,");
+      "List of expressions and output field names"
+        + " is not complete, each expression must explicitly give and output name,");
     List<NamedExpression> ret = new ArrayList<>();
     for (int i = 0; i < expressionsAndOutputNames.length; i += 2) {
-      ret.add(new NamedExpression(parseExpr(expressionsAndOutputNames[i]),
+      ret.add(new NamedExpression(LogicalExpressionParser.parse(expressionsAndOutputNames[i]),
         new FieldReference(new SchemaPath(new PathSegment.NameSegment(expressionsAndOutputNames[i + 1])))));
     }
     return ret;
   }
 
-  public static LogicalExpression parseExpr(String expr) {
-    ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
-    CommonTokenStream tokens = new CommonTokenStream(lexer);
-    ExprParser parser = new ExprParser(tokens);
-    try {
-      return parser.parse().e;
-    } catch (RecognitionException e) {
-      throw new RuntimeException("Error parsing expression: " + expr);
-    }
-  }
 }

exec/java-exec/src/test/java/org/apache/drill/test/PhysicalOpUnitTestBase.java

@@ -29,7 +29,7 @@
 import org.apache.drill.exec.ops.FragmentStats;
 import org.apache.drill.exec.physical.base.PhysicalVisitor;
 import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.planner.logical.DrillLogicalTestutils;
+import org.apache.drill.exec.planner.logical.DrillLogicalTestUtils;
 import org.apache.drill.exec.proto.CoordinationProtos;
 import org.apache.drill.exec.rpc.control.Controller;
 import org.apache.drill.exec.rpc.control.WorkEventBus;
@@ -42,7 +42,6 @@
 import org.apache.drill.exec.work.filter.RuntimeFilterWritable;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.logical.data.JoinCondition;
 import org.apache.drill.common.logical.data.NamedExpression;
@@ -115,21 +114,16 @@ public void teardown() {
     scanDecodeExecutor.shutdownNow();
   }
 
-  @Override
-  protected LogicalExpression parseExpr(String expr) {
-    return DrillLogicalTestutils.parseExpr(expr);
-  }
-
   protected Order.Ordering ordering(String expression, RelFieldCollation.Direction direction, RelFieldCollation.NullDirection nullDirection) {
-    return DrillLogicalTestutils.ordering(expression, direction, nullDirection);
+    return DrillLogicalTestUtils.ordering(expression, direction, nullDirection);
   }
 
   protected JoinCondition joinCond(String leftExpr, String relationship, String rightExpr) {
-    return DrillLogicalTestutils.joinCond(leftExpr, relationship, rightExpr);
+    return DrillLogicalTestUtils.joinCond(leftExpr, relationship, rightExpr);
   }
 
   protected List<NamedExpression> parseExprs(String... expressionsAndOutputNames) {
-    return DrillLogicalTestutils.parseExprs(expressionsAndOutputNames);
+    return DrillLogicalTestUtils.parseExprs(expressionsAndOutputNames);
   }
 
   protected static class BatchIterator implements Iterable<VectorAccessible> {