Skip to content

Commit

Permalink
QL: Introduce ParserUtils to consolidate code (#76399) (#76402)
Browse files Browse the repository at this point in the history
Move common ANTLR utility methods, such as extracting code or case
insensitive streams into QL.
Replace CaseInsensitiveStream (which relies on ANTLRInputStream which
has been deprecated) with a CharStream variant.
  • Loading branch information
costin committed Aug 12, 2021
1 parent 38b00c8 commit e1d6ee1
Show file tree
Hide file tree
Showing 13 changed files with 252 additions and 227 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,11 @@
package org.elasticsearch.xpack.eql.parser;

import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.elasticsearch.xpack.ql.parser.ParserUtils;
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.ql.tree.Location;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.util.Check;

import java.util.ArrayList;
import java.util.List;

/**
Expand All @@ -26,90 +21,19 @@ abstract class AbstractBuilder extends EqlBaseBaseVisitor<Object> {

@Override
public Object visit(ParseTree tree) {
Object result = super.visit(tree);
Check.notNull(result, "Don't know how to handle context [{}] with value [{}]", tree.getClass(), tree.getText());
return result;
return ParserUtils.visit(super::visit, tree);
}

@SuppressWarnings("unchecked")
protected <T> T typedParsing(ParseTree ctx, Class<T> type) {
Object result = ctx.accept(this);
if (type.isInstance(result)) {
return (T) result;
}

throw new ParsingException(source(ctx), "Invalid query '{}'[{}] given; expected {} but found {}",
ctx.getText(), ctx.getClass().getSimpleName(),
type.getSimpleName(), (result != null ? result.getClass().getSimpleName() : "null"));
return ParserUtils.typedParsing(this, ctx, type);
}

protected LogicalPlan plan(ParseTree ctx) {
return typedParsing(ctx, LogicalPlan.class);
}

protected List<LogicalPlan> plans(List<? extends ParserRuleContext> ctxs) {
return visitList(ctxs, LogicalPlan.class);
}

protected <T> List<T> visitList(List<? extends ParserRuleContext> contexts, Class<T> clazz) {
List<T> results = new ArrayList<>(contexts.size());
for (ParserRuleContext context : contexts) {
results.add(clazz.cast(visit(context)));
}
return results;
}

static Source source(ParseTree ctx) {
if (ctx instanceof ParserRuleContext) {
return source((ParserRuleContext) ctx);
}
return Source.EMPTY;
}

static Source source(TerminalNode terminalNode) {
Check.notNull(terminalNode, "terminalNode is null");
return source(terminalNode.getSymbol());
}

static Source source(ParserRuleContext parserRuleContext) {
Check.notNull(parserRuleContext, "parserRuleContext is null");
Token start = parserRuleContext.start;
Token stop = parserRuleContext.stop != null ? parserRuleContext.stop : start;
Interval interval = new Interval(start.getStartIndex(), stop.getStopIndex());
String text = start.getInputStream().getText(interval);
return new Source(new Location(start.getLine(), start.getCharPositionInLine()), text);
}

static Source source(Token token) {
Check.notNull(token, "token is null");
String text = token.getInputStream().getText(new Interval(token.getStartIndex(), token.getStopIndex()));
return new Source(new Location(token.getLine(), token.getCharPositionInLine()), text);
}

Source source(ParserRuleContext begin, ParserRuleContext end) {
Check.notNull(begin, "begin is null");
Check.notNull(end, "end is null");
Token start = begin.start;
Token stop = end.stop != null ? end.stop : begin.stop;
Interval interval = new Interval(start.getStartIndex(), stop.getStopIndex());
String text = start.getInputStream().getText(interval);
return new Source(new Location(start.getLine(), start.getCharPositionInLine()), text);
}

static Source source(TerminalNode begin, ParserRuleContext end) {
Check.notNull(begin, "begin is null");
Check.notNull(end, "end is null");
Token start = begin.getSymbol();
Token stop = end.stop != null ? end.stop : start;
String text = start.getInputStream().getText(new Interval(start.getStartIndex(), stop.getStopIndex()));
return new Source(new Location(start.getLine(), start.getCharPositionInLine()), text);
}

/**
* Retrieves the raw text of the node (without interpreting it as a string literal).
*/
static String text(ParseTree node) {
return node == null ? null : node.getText();
return ParserUtils.visitList(this, ctxs, LogicalPlan.class);
}

public static String unquoteString(Source source) {
Expand Down Expand Up @@ -212,11 +136,4 @@ private static void checkForSingleQuotedString(Source source, String text, int i
"Use double quotes [\"] to define string literals, not single quotes [']");
}
}

@Override
public Object visitTerminal(TerminalNode node) {
Source source = source(node);
throw new ParsingException(source, "Does not know how to handle {}", source.text());
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@

import static java.util.Collections.emptyList;
import static java.util.stream.Collectors.toList;
import static org.elasticsearch.xpack.ql.parser.ParserUtils.source;
import static org.elasticsearch.xpack.ql.parser.ParserUtils.visitList;


public class ExpressionBuilder extends IdentifierBuilder {
Expand All @@ -73,7 +75,7 @@ protected Expression expression(ParseTree ctx) {
}

protected List<Expression> expressions(List<? extends ParserRuleContext> contexts) {
return visitList(contexts, Expression.class);
return visitList(this, contexts, Expression.class);
}

@Override
Expand All @@ -84,7 +86,7 @@ public Expression visitSingleExpression(EqlBaseParser.SingleExpressionContext ct
@Override
public List<Attribute> visitJoinKeys(JoinKeysContext ctx) {
try {
return ctx != null ? visitList(ctx.expression(), Attribute.class) : emptyList();
return ctx != null ? visitList(this, ctx.expression(), Attribute.class) : emptyList();
} catch (ClassCastException ex) {
Source source = source(ctx);
throw new ParsingException(source, "Unsupported join key ", source.text());
Expand Down Expand Up @@ -203,7 +205,7 @@ private Expression combineExpressions(
List<? extends ParserRuleContext> expressions,
java.util.function.Function<Expression, Expression> mapper
) {
return Predicates.combineOr(expressions(expressions).stream().map(mapper::apply).collect(toList()));
return Predicates.combineOr(expressions(expressions).stream().map(mapper).collect(toList()));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.IdentifierContext;
import org.elasticsearch.xpack.eql.parser.EqlBaseParser.QualifiedNameContext;

import static org.elasticsearch.xpack.ql.parser.ParserUtils.visitList;

abstract class IdentifierBuilder extends AbstractBuilder {

@Override
Expand All @@ -24,7 +26,7 @@ public String visitQualifiedName(QualifiedNameContext ctx) {
}

// this is fine, because we've already checked for array indexes [...]
return Strings.collectionToDelimitedString(visitList(ctx.identifier(), String.class), ".");
return Strings.collectionToDelimitedString(visitList(this, ctx.identifier(), String.class), ".");
}

private static String unquoteIdentifier(String identifier) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@

import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.ql.parser.ParserUtils.source;
import static org.elasticsearch.xpack.ql.parser.ParserUtils.text;
import static org.elasticsearch.xpack.ql.tree.Source.synthetic;

public abstract class LogicalPlanBuilder extends ExpressionBuilder {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.ql.parser;

import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.misc.Interval;

// Wrapping stream for handling case-insensitive grammars

// This approach is taken from the ANTLR documentation
// https://github.com/antlr/antlr4/blob/master/doc/resources/CaseChangingCharStream.java
// https://github.com/antlr/antlr4/blob/master/doc/case-insensitive-lexing.md

/**
* This class supports case-insensitive lexing by wrapping an existing
* {@link CharStream} and forcing the lexer to see either upper or
* lowercase characters. Grammar literals should then be either upper or
* lower case such as 'BEGIN' or 'begin'. The text of the character
* stream is unaffected. Example: input 'BeGiN' would match lexer rule
* 'BEGIN' if constructor parameter upper=true but getText() would return
* 'BeGiN'.
*/
public class CaseChangingCharStream implements CharStream {

private final CharStream stream;
private final boolean upper;

/**
* Constructs a new CaseChangingCharStream wrapping the given {@link CharStream} forcing
* all characters to upper case or lower case.
* @param stream The stream to wrap.
* @param upper If true force each symbol to upper case, otherwise force to lower.
*/
public CaseChangingCharStream(CharStream stream, boolean upper) {
this.stream = stream;
this.upper = upper;
}

@Override
public String getText(Interval interval) {
return stream.getText(interval);
}

@Override
public void consume() {
stream.consume();
}

@Override
public int LA(int i) {
int c = stream.LA(i);
if (c <= 0) {
return c;
}
return upper ? Character.toUpperCase(c) : Character.toLowerCase(c);
}

@Override
public int mark() {
return stream.mark();
}

@Override
public void release(int marker) {
stream.release(marker);
}

@Override
public int index() {
return stream.index();
}

@Override
public void seek(int index) {
stream.seek(index);
}

@Override
public int size() {
return stream.size();
}

@Override
public String getSourceName() {
return stream.getSourceName();
}
}

0 comments on commit e1d6ee1

Please sign in to comment.