DRILL-5956: Add Storage Plugin for Apache Druid

apache · Jun 23, 2020 · a7f2c8f · a7f2c8f
1 parent 1b95c0a
commit a7f2c8f
Show file tree

Hide file tree

Showing 67 changed files with 4,887 additions and 16 deletions.
diff --git a/common/src/test/java/org/apache/drill/categories/DruidStorageTest.java b/common/src/test/java/org/apache/drill/categories/DruidStorageTest.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.categories;
+
+/**
+ * This is a category used to mark unit tests that test the Druid storage plugin.
+ */
+public interface DruidStorageTest {
+}
diff --git a/contrib/pom.xml b/contrib/pom.xml
@@ -56,6 +56,7 @@
     <module>storage-kudu</module>
     <module>storage-opentsdb</module>
     <module>storage-http</module>
+    <module>storage-druid</module>
   </modules>
 
 </project>
diff --git a/contrib/storage-druid/README.md b/contrib/storage-druid/README.md
@@ -0,0 +1,59 @@
+# Drill Apache Druid Plugin
+
+Drill druid storage plugin allows you to perform SQL queries against Druid datasource(s).
+This storage plugin is part of [Apache Drill](https://github.com/apache/drill)
+
+### Tested with Druid version
+[0.16.0-incubating](https://github.com/apache/incubator-druid/releases/tag/druid-0.16.0-incubating)
+
+### Druid API
+
+Druid supports multiple native queries to address sundry use-cases.
+To fetch raw druid rows, druid API support two forms of query, `SELECT` (no relation to SQL) and `SCAN`.
+Currently, this plugin uses the [Select](https://druid.apache.org/docs/latest/querying/select-query.html)
+query API to fetch raw rows from druid as json.
+
+### Filter Push-Down
+
+Filters pushed down to native druid filter structure, converting SQL where clauses to the respective druid [Filters](https://druid.apache.org/docs/latest/querying/filters.html).
+
+### Plugin Registration
+
+The plugin can be registered in Apache Drill using the drill web interface by navigating to the ```storage``` page.
+Following is the default registration configuration.
+```json
+{
+  "type" : "druid",
+  "brokerAddress" : "http://localhost:8082",
+  "coordinatorAddress": "http://localhost:8081",
+  "averageRowSizeBytes": 100,
+  "enabled" : false
+}
+```
+
+### Druid storage plugin developer notes.
+
+* Building the plugin 
+
+    `mvn install -pl contrib/storage-druid`
+
+* Building DRILL
+
+    `mvn clean install -DskipTests`
+
+* Start Drill In Embedded Mode (mac)
+
+    ```shell script
+    distribution/target/apache-drill-1.18.0-SNAPSHOT/apache-drill-1.18.0-SNAPSHOT/bin/drill-embedded
+    ```
+
+* Starting Druid (Docker and Docker Compose required)
+    ```
+    cd contrib/storage-druid/src/test/resources/druid
+    docker-compose up -d
+    ```
+
+  * There is an `Indexing Task Json` in the same folder as the docker compose file. It can be used to ingest the wikipedia datasource.
+
+  * Make sure the druid storage plugin is enabled in Drill.
+
diff --git a/contrib/storage-druid/pom.xml b/contrib/storage-druid/pom.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>drill-contrib-parent</artifactId>
+        <groupId>org.apache.drill.contrib</groupId>
+        <version>1.18.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>drill-druid-storage</artifactId>
+    <name>contrib/druid-storage-plugin</name>
+    <properties>
+        <druid.TestSuite>**/DruidTestSuit.class</druid.TestSuite>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.drill.exec</groupId>
+            <artifactId>drill-java-exec</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>org.apache.drill.exec</groupId>
+            <artifactId>drill-java-exec</artifactId>
+            <classifier>tests</classifier>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.drill</groupId>
+            <artifactId>drill-common</artifactId>
+            <classifier>tests</classifier>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <!-- use 2.9.1 for Java 7 projects -->
+            <version>3.11.1</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <includes>
+                        <include>${druid.TestSuite}</include>
+                    </includes>
+                    <excludes>
+                        <exclude>**/TestDruidQueries.java</exclude>
+                    </excludes>
+                    <systemProperties>
+                        <property>
+                            <name>logback.log.dir</name>
+                            <value>${project.build.directory}/surefire-reports</value>
+                        </property>
+                    </systemProperties>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>