Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DRILL-8454: Disable unsupported MapR profile and plugin #2825

Merged
merged 1 commit into from
Oct 21, 2023
Merged

DRILL-8454: Disable unsupported MapR profile and plugin #2825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 0 additions & 2 deletions common/src/main/java/org/apache/drill/common/config/DrillProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,6 @@ public final class DrillProperties extends Properties {
public static final String TLS_HANDSHAKE_TIMEOUT = "TLSHandshakeTimeout";
public static final String TLS_PROVIDER = "TLSProvider";
public static final String USE_SYSTEM_TRUSTSTORE = "useSystemTrustStore";
public static final String USE_MAPR_SSL_CONFIG = "useMapRSSLConfig";

public static final String QUERY_TAGS = "queryTags";

// Although all properties from the application are sent to the server (from the client), the following
Expand Down
36 changes: 18 additions & 18 deletions contrib/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,40 +39,40 @@
</dependencies>

<modules>
<!-- Please keep this list sorted. -->
<module>data</module>
<module>udfs</module>
<module>storage-hbase</module>
<module>format-maprdb</module>
<module>format-syslog</module>
<module>format-ltsv</module>
<module>format-access</module>
<module>format-deltalake</module>
<module>format-esri</module>
<module>format-excel</module>
<module>format-hdf5</module>
<module>format-httpd</module>
<module>format-esri</module>
<module>format-iceberg</module>
<module>format-image</module>
<module>format-log</module>
<module>format-access</module>
<module>format-ltsv</module>
<module>format-pcapng</module>
<module>format-pdf</module>
<module>format-hdf5</module>
<module>format-sas</module>
<module>format-spss</module>
<module>format-syslog</module>
<module>format-xml</module>
<module>format-image</module>
<module>format-pcapng</module>
<module>format-iceberg</module>
<module>format-deltalake</module>
<module>storage-cassandra</module>
<module>storage-drill</module>
<module>storage-phoenix</module>
<module>storage-druid</module>
<module>storage-elasticsearch</module>
<module>storage-googlesheets</module>
<module>storage-hbase</module>
<module>storage-hive</module>
<module>storage-mongo</module>
<module>storage-http</module>
<module>storage-jdbc</module>
<module>storage-kafka</module>
<module>storage-kudu</module>
<module>storage-mongo</module>
<module>storage-opentsdb</module>
<module>storage-phoenix</module>
<module>storage-splunk</module>
<module>storage-http</module>
<module>storage-druid</module>
<module>storage-elasticsearch</module>
<module>storage-cassandra</module>
<module>udfs</module>
</modules>

</project>
55 changes: 0 additions & 55 deletions contrib/storage-hbase/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,60 +258,5 @@
</dependency>
</dependencies>
</profile>

<profile>
<id>mapr</id>
<properties>
<alt-hadoop>mapr</alt-hadoop>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-testing-util</artifactId>
<scope>test</scope>
<classifier>tests</classifier>
<exclusions>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.tdunning</groupId>
<artifactId>json</artifactId>
</dependency>
</dependencies>
</profile>
</profiles>
</project>
50 changes: 0 additions & 50 deletions contrib/storage-hive/core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,54 +321,4 @@
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>mapr</id>
<properties>
<freemarker.conf.file>src/main/codegen/config.fmpp</freemarker.conf.file>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>add-mapr-sources</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>scrMapr/main/java</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.drill.contrib</groupId>
<artifactId>drill-format-mapr</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.tdunning</groupId>
<artifactId>json</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-maprdb-json-handler</artifactId>
</dependency>
<dependency>
<groupId>com.mapr.db</groupId>
<artifactId>maprdb-mapreduce</artifactId>
<scope>runtime</scope>
</dependency>
</dependencies>
</profile>
</profiles>
</project>
40 changes: 0 additions & 40 deletions ...b/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ops.OptimizerRulesContext;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
Expand All @@ -50,11 +48,9 @@
import org.apache.drill.exec.store.AbstractStoragePlugin;
import org.apache.drill.exec.store.SchemaConfig;
import org.apache.drill.exec.store.StoragePluginOptimizerRule;
import org.apache.drill.exec.store.dfs.FormatPlugin;
import org.apache.drill.exec.store.hive.schema.HiveSchemaFactory;
import com.google.common.collect.ImmutableSet;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.MetaException;
Expand All @@ -64,8 +60,6 @@ public class HiveStoragePlugin extends AbstractStoragePlugin {

private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveStoragePlugin.class);

public static final String HIVE_MAPRDB_FORMAT_PLUGIN_NAME = "hive-maprdb";

private final HiveStoragePluginConfig config;
private HiveSchemaFactory schemaFactory;
private final HiveConf hiveConf;
Expand Down Expand Up @@ -206,44 +200,10 @@ public Set<StoragePluginOptimizerRule> getOptimizerRules(OptimizerRulesContext o
options.getBoolean(ExecConstants.HIVE_OPTIMIZE_PARQUET_SCAN_WITH_NATIVE_READER)) {
ruleBuilder.add(ConvertHiveParquetScanToDrillParquetScan.INSTANCE);
}
if (options.getBoolean(ExecConstants.HIVE_OPTIMIZE_MAPRDB_JSON_SCAN_WITH_NATIVE_READER)) {
try {
Class<?> hiveToDrillMapRDBJsonRuleClass =
Class.forName("org.apache.drill.exec.planner.sql.logical.ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan");
ruleBuilder.add((StoragePluginOptimizerRule) hiveToDrillMapRDBJsonRuleClass.getField("INSTANCE").get(null));
} catch (ReflectiveOperationException e) {
logger.warn("Current Drill build is not designed for working with Hive MapR-DB tables. " +
"Please disable {} option", ExecConstants.HIVE_OPTIMIZE_MAPRDB_JSON_SCAN_WITH_NATIVE_READER);
}
}
return ruleBuilder.build();
}
default:
return ImmutableSet.of();
}
}

@Override
public FormatPlugin getFormatPlugin(FormatPluginConfig formatConfig) {
// TODO: implement formatCreator similar to FileSystemPlugin formatCreator. DRILL-6621
try {
Class<?> mapRDBFormatPluginConfigClass =
Class.forName("org.apache.drill.exec.store.mapr.db.MapRDBFormatPluginConfig");
Class<?> mapRDBFormatPluginClass =
Class.forName("org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin");

if (mapRDBFormatPluginConfigClass.isInstance(formatConfig)) {
return (FormatPlugin) mapRDBFormatPluginClass.getConstructor(
new Class[]{String.class, DrillbitContext.class, Configuration.class,
StoragePluginConfig.class, mapRDBFormatPluginConfigClass})
.newInstance(
new Object[]{HIVE_MAPRDB_FORMAT_PLUGIN_NAME, context, hiveConf, config, formatConfig});
}
} catch (ReflectiveOperationException e) {
throw new DrillRuntimeException("The error is occurred while connecting to MapR-DB or instantiating mapRDBFormatPlugin", e);
}
throw new DrillRuntimeException(String.format("Hive storage plugin doesn't support usage of %s format plugin",
formatConfig.getClass().getName()));
}

}
97 changes: 2 additions & 95 deletions distribution/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,17 +453,6 @@
<artifactId>drill-format-esri</artifactId>
<version>${project.version}</version>
</dependency>
<!--
drill-format-mapr is intentionally excluded from distribution for the default profile
to avoid adding mapr-specific jars into jars/3rdparty folder and inflation of the target jar.
For more details please see DRILL-7213 and DRILL-6929.

<dependency>
<groupId>org.apache.drill.contrib</groupId>
<artifactId>drill-format-mapr</artifactId>
<version>${project.version}</version>
</dependency>
-->
<dependency>
<groupId>org.apache.drill.contrib</groupId>
<artifactId>drill-format-excel</artifactId>
Expand Down Expand Up @@ -499,6 +488,7 @@
<artifactId>drill-storage</artifactId>
<version>${project.version}</version>
</dependency>

</dependencies>
</profile>

Expand Down Expand Up @@ -535,90 +525,6 @@
</plugins>
</build>
</profile>
<profile>
<id>mapr</id>
<dependencies>
<dependency>
<groupId>org.apache.drill.contrib</groupId>
<artifactId>drill-format-mapr</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>com.mapr</groupId>
<artifactId>mapr-test-annotations</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-annotations</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-protocol</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-maprdb-json-handler</artifactId>
</dependency>
<dependency>
<groupId>com.mapr.db</groupId>
<artifactId>maprdb-mapreduce</artifactId>
</dependency>
<dependency>
<groupId>com.mapr.security</groupId>
<artifactId>mapr-security-web</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>copy-winutils</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.6.0</version>
<executions>
<execution>
<id>unpack-winutils</id>
<goals>
<goal>unpack</goal>
</goals>
<phase>process-resources</phase>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-winutils</artifactId>
<version>2.7.0-mapr-1506</version>
<type>zip</type>
<overWrite>true</overWrite>
<outputDirectory>${project.build.directory}/winutils</outputDirectory>
<excludes>**/*.pdb,**/*.lib,**/*.exp</excludes>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>rpm</id>
<build>
Expand Down Expand Up @@ -868,6 +774,7 @@
</plugins>
</build>
</profile>

</profiles>

</project>