[PPL] add basic parser support

build.gradle

@@ -56,18 +56,7 @@ allprojects {
 
 apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.testclusters'
-// Todo, removed after migrate SQL to subProject
-project('plugin') {
-    apply plugin: 'elasticsearch.esplugin'
-}
-// Todo, removed after migrate SQL to subProject
-project('integ-test') {
-    apply plugin: 'elasticsearch.testclusters'
-}
 apply plugin: 'jacoco'
-if (!System.properties.containsKey('tests.rest.cluster') && !System.properties.containsKey('tests.cluster')){
-    apply from: 'build-tools/sqlplugin-coverage.gradle'
-}
 apply plugin: 'antlr'
 
 jacoco {

common/build.gradle

@@ -0,0 +1,14 @@
+plugins {
+    id 'java'
+    id "io.freefair.lombok"
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    compile "org.antlr:antlr4-runtime:4.7.1"
+
+    testCompile group: 'junit', name: 'junit', version: '4.12'
+}

common/lombok.config

@@ -0,0 +1,2 @@
+# This file is generated by the 'io.freefair.lombok' Gradle plugin
+config.stopBubbling = true

common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/CaseInsensitiveCharStream.java

@@ -0,0 +1,82 @@
+/*
+ *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.common.antlr;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.CharStreams;
+import org.antlr.v4.runtime.misc.Interval;
+
+/**
+ * Custom stream to convert character to upper case for case insensitive grammar before sending to lexer.
+ */
+public class CaseInsensitiveCharStream implements CharStream {
+
+    /** Character stream */
+    private final CharStream charStream;
+
+    public CaseInsensitiveCharStream(String sql) {
+        this.charStream = CharStreams.fromString(sql);
+    }
+
+    @Override
+    public String getText(Interval interval) {
+        return charStream.getText(interval);
+    }
+
+    @Override
+    public void consume() {
+        charStream.consume();
+    }
+
+    @Override
+    public int LA(int i) {
+        int c = charStream.LA(i);
+        if (c <= 0) {
+            return c;
+        }
+        return Character.toUpperCase(c);
+    }
+
+    @Override
+    public int mark() {
+        return charStream.mark();
+    }
+
+    @Override
+    public void release(int marker) {
+        charStream.release(marker);
+    }
+
+    @Override
+    public int index() {
+        return charStream.index();
+    }
+
+    @Override
+    public void seek(int index) {
+        charStream.seek(index);
+    }
+
+    @Override
+    public int size() {
+        return charStream.size();
+    }
+
+    @Override
+    public String getSourceName() {
+        return charStream.getSourceName();
+    }
+}

common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/antlr/SyntaxAnalysisErrorListener.java

@@ -0,0 +1,74 @@
+/*
+ *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.common.antlr;
+
+import org.antlr.v4.runtime.BaseErrorListener;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.Recognizer;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.misc.IntervalSet;
+
+import java.util.Locale;
+
+/**
+ * Syntax analysis error listener that handles any syntax error by throwing exception with useful information.
+ */
+public class SyntaxAnalysisErrorListener extends BaseErrorListener {
+
+    @Override
+    public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
+                            int line, int charPositionInLine, String msg,
+                            RecognitionException e) {
+
+        CommonTokenStream tokens = (CommonTokenStream) recognizer.getInputStream();
+        Token offendingToken = (Token) offendingSymbol;
+        String query = tokens.getText();
+
+        throw new RuntimeException(
+                String.format(Locale.ROOT,
+                        "Failed to parse query due to offending symbol [%s] at: '%s' <--- HERE... More details: %s",
+                        getOffendingText(offendingToken),
+                        truncateQueryAtOffendingToken(query, offendingToken),
+                        getDetails(recognizer, msg, e)
+                )
+        );
+    }
+
+    private String getOffendingText(Token offendingToken) {
+        return offendingToken.getText();
+    }
+
+    private String truncateQueryAtOffendingToken(String query, Token offendingToken) {
+        return query.substring(0, offendingToken.getStopIndex() + 1);
+    }
+
+    /**
+     * As official JavaDoc says, e=null means parser was able to recover from the error.
+     * In other words, "msg" argument includes the information we want.
+     */
+    private String getDetails(Recognizer<?, ?> recognizer, String msg, RecognitionException e) {
+        String details;
+        if (e == null) {
+            details = msg;
+        } else {
+            IntervalSet followSet = e.getExpectedTokens();
+            details = "Expecting tokens in " + followSet.toString(recognizer.getVocabulary());
+        }
+        return details;
+    }
+
+}

elasticsearch/build.gradle

@@ -0,0 +1,10 @@
+apply plugin: 'java'
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    compile group: 'org.elasticsearch', name: 'elasticsearch', version: "${es_version}"
+    testCompile group: 'junit', name: 'junit', version: '4.12'
+}

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/security/SecurityAccess.java

@@ -0,0 +1,38 @@
+/*
+ *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.elasticsearch.security;
+
+import org.elasticsearch.SpecialPermission;
+
+import java.io.IOException;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * Ref: https://www.elastic.co/guide/en/elasticsearch/plugins/current/plugin-authors.html#_java_security_permissions
+ */
+public class SecurityAccess {
+
+    public static <T> T doPrivileged(final PrivilegedExceptionAction<T> operation) throws IOException {
+        SpecialPermission.check();
+        try {
+            return AccessController.doPrivileged(operation);
+        } catch (final PrivilegedActionException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+}

integ-test/build.gradle

@@ -23,6 +23,7 @@ dependencies {
 dependencyLicenses.enabled = false
 testingConventions.enabled = false
 checkstyleTest.ignoreFailures = true
+forbiddenApisTest.enabled = false
 
 tasks.integTest.dependsOn(':plugin:bundlePlugin')
 testClusters.integTest {

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/PPLPluginIT.java

@@ -17,16 +17,57 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.test.rest.ESRestTestCase;
+import org.hamcrest.Description;
+import org.hamcrest.TypeSafeMatcher;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 import java.io.IOException;
+import java.util.Locale;
 
-import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.hasProperty;
 
 public class PPLPluginIT extends ESRestTestCase {
+    @Rule
+    public ExpectedException exceptionRule = ExpectedException.none();
 
+    @Test
     public void testQueryEndpointShouldOK() throws IOException {
-        Response response = client().performRequest(new Request("POST", "/_opendistro/_ppl"));
-        assertThat(response.getStatusLine().getStatusCode(), is(200));
+        Response response = client().performRequest(makeRequest("search source=a"));
+        assertThat(response, statusCode(200));
+    }
+
+    @Test
+    public void testQueryEndpointShouldFail() throws IOException {
+        exceptionRule.expect(ResponseException.class);
+        exceptionRule.expect(hasProperty("response", statusCode(500)));
+
+        client().performRequest(makeRequest("search invalid"));
+    }
+
+    protected Request makeRequest(String query) {
+        Request post = new Request("POST", "/_opendistro/_ppl");
+        post.setJsonEntity(String.format(Locale.ROOT,
+                "{\n" +
+                        "  \"query\": \"%s\"\n" +
+                        "}", query));
+        return post;
+    }
+
+    private TypeSafeMatcher<Response> statusCode(int statusCode) {
+        return new TypeSafeMatcher<Response>() {
+            @Override
+            public void describeTo(Description description) {
+                description.appendText(String.format(Locale.ROOT, "statusCode=%d", statusCode));
+            }
+
+            @Override
+            protected boolean matchesSafely(Response resp) {
+                return resp.getStatusLine().getStatusCode() == statusCode;
+            }
+        };
     }
 }

plugin/build.gradle

@@ -1,5 +1,6 @@
 apply plugin: 'java'
 apply plugin: 'idea'
+apply plugin: 'elasticsearch.esplugin'
 
 ext {
     projectSubstitutions = [:]
@@ -20,7 +21,15 @@ esplugin {
 test.enabled = false
 integTest.enabled = false
 dependencyLicenses.enabled = false
+thirdPartyAudit.enabled = false
+
+configurations.all {
+    // conflict with spring-jcl
+    exclude group: "commons-logging", module: "commons-logging"
+}
 
 dependencies {
+    compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE'
     compile project(":ppl")
+    compile project(':elasticsearch')
 }

plugin/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/request/PPLQueryRequestFactory.java

@@ -0,0 +1,58 @@
+/*
+ *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
+package com.amazon.opendistroforelasticsearch.sql.plugin.request;
+
+import com.amazon.opendistroforelasticsearch.sql.ppl.domain.PPLQueryRequest;
+import org.elasticsearch.rest.RestRequest;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class PPLQueryRequestFactory {
+    private static final String PPL_URL_PARAM_KEY = "ppl";
+    private static final String PPL_FIELD_NAME = "query";
+
+    public static PPLQueryRequest getPPLRequest(RestRequest request) {
+        switch (request.method()) {
+            case GET:
+                return parsePPLRequestFromUrl(request);
+            case POST:
+                return parsePPLRequestFromPayload(request);
+            default:
+                throw new IllegalArgumentException("ES PPL doesn't supported HTTP " + request.method().name());
+        }
+    }
+
+    private static PPLQueryRequest parsePPLRequestFromUrl(RestRequest restRequest) {
+        String ppl;
+
+        ppl = restRequest.param(PPL_URL_PARAM_KEY);
+        if (ppl == null) {
+            throw new IllegalArgumentException("Cannot find ppl parameter from the URL");
+        }
+        return new PPLQueryRequest(ppl, null);
+    }
+
+    private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restRequest) {
+        String content = restRequest.content().utf8ToString();
+        JSONObject jsonContent;
+        try {
+            jsonContent = new JSONObject(content);
+        } catch (JSONException e) {
+            throw new IllegalArgumentException("Failed to parse request payload", e);
+        }
+        return new PPLQueryRequest(jsonContent.getString(PPL_FIELD_NAME), jsonContent);
+    }
+}