Skip to content
Permalink
Browse files
DRILL-6494: Drill Plugins Handler
- Storage Plugins Handler service is used op the Drill start-up stage and it updates storage plugins configs from
  storage-plugins-override.conf file. If plugins configs are present in the persistence store - they are updated,
  otherwise bootstrap plugins are updated and the result configs are loaded to persistence store. If the enabled
  status is absent in the storage-plugins-override.conf file, the last plugin config enabled status persists.
- 'drill.exec.storage.action_on_plugins_override_file' Boot option is added. This is the action, which should be
  performed on the storage-plugins-override.conf file after successful updating storage plugins configs.
  Possible values are: "none" (default), "rename" and "remove".
- The "NULL" issue with updating Hive plugin config by REST is solved. But clients are still being instantiated for disabled
  plugins - DRILL-6412.
- "org.honton.chas.hocon:jackson-dataformat-hocon" library is added for the proper deserializing HOCON conf file
- additional refactoring: "com.typesafe:config" and "org.apache.commons:commons-lang3" are placed into DependencyManagement
  block with proper versions; correct properties for metrics in "drill-override-example.conf" are specified

closes #1345
  • Loading branch information
vdiravka committed Jul 3, 2018
1 parent 069c304 commit 0ae7035de390c699f574d1ec25d45cd8b20a8b94
Showing 35 changed files with 680 additions and 201 deletions.
@@ -53,13 +53,11 @@
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.0.0</version>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.1</version>
</dependency>

<dependency>
@@ -31,4 +31,7 @@ public interface CommonConstants {
/** Override configuration file name. (Classpath resource pathname.) */
String CONFIG_OVERRIDE_RESOURCE_PATHNAME = "drill-override.conf";

/** Override plugins configs file name. (Classpath resource pathname.) */
String STORAGE_PLUGINS_OVERRIDE_CONF = "storage-plugins-override.conf";

}
@@ -261,7 +261,7 @@ public <T> Class<T> getClassAt(String location, Class<T> clazz) throws DrillConf
final String className = getString(location);
if (className == null) {
throw new DrillConfigurationException(String.format(
"No class defined at location '%s'. Expected a definition of the class []",
"No class defined at location '%s'. Expected a definition of the class [%s]",
location, clazz.getCanonicalName()));
}

@@ -51,7 +51,6 @@
import com.google.common.base.Stopwatch;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;

import javassist.bytecode.AccessFlag;
import javassist.bytecode.AnnotationsAttribute;
@@ -320,15 +319,12 @@ public static Collection<URL> getConfigURLs() {
* to scan for (relative to specified class loaders' classpath roots)
* @param returnRootPathname whether to collect classpath root portion of
* URL for each resource instead of full URL of each resource
* @param classLoaders set of class loaders in which to look up resource;
* none (empty array) to specify to use current thread's context
* class loader and {@link Reflections}'s class loader
* @returns ...; empty set if none
*/
public static Set<URL> forResource(final String resourcePathname, final boolean returnRootPathname) {
logger.debug("Scanning classpath for resources with pathname \"{}\".",
resourcePathname);
final Set<URL> resultUrlSet = Sets.newHashSet();
final Set<URL> resultUrlSet = new HashSet<>();
final ClassLoader classLoader = ClassPathScanner.class.getClassLoader();
try {
final Enumeration<URL> resourceUrls = classLoader.getResources(resourcePathname);
@@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.drill.exec.util;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.Date;

/**
* It defines possible actions on the file and performs the necessary action
*/
public enum ActionOnFile {

/**
* No action will be performed
*/
NONE {
@Override
public void action(URL url) { }
},

/**
* Rename the file by adding current timestamp value with "yyyyMMdd_HHmmss" format before last dot of original file name<p>
* Example:<br>
* Original file name: "storage-plugins-override.conf"<br>
* New file name: "storage-plugins-override-20180703_033354.conf"
*/
RENAME {
@Override
public void action(URL url) {
String fileName = url.getFile();
File file = new File(url.getPath());
String currentDateTime = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date());
String newFileName = new StringBuilder(fileName)
.insert(fileName.lastIndexOf("."), "-" + currentDateTime)
.toString();
Path filePath = file.toPath();
try {
Files.move(filePath, filePath.resolveSibling(newFileName));
} catch (IOException e) {
logger.error("There was an error during file {} rename.", fileName, e);
}
}
},

/**
* It removes the file
*/
REMOVE {
@Override
public void action(URL url) {
File file = new File(url.getPath());
try {
Files.delete(file.toPath());
} catch (IOException e) {
logger.error("There was an error during file {} removing.", url.getFile(), e);
}
}
};

private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ActionOnFile.class);

/**
* This is an action which should be performed on the file
* @param url the file URL
*/
public abstract void action(URL url);
}
@@ -2,11 +2,11 @@
"storage":{
hbase : {
type:"hbase",
enabled: false,
config : {
"hbase.zookeeper.quorum" : "localhost",
"hbase.zookeeper.property.clientPort" : 2181
}
},
enabled: false
}
}
}
@@ -27,7 +27,6 @@
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
@@ -72,6 +71,7 @@ public HiveSchemaFactory(final HiveStoragePlugin plugin, final String name, fina
isDrillImpersonationEnabled = plugin.getContext().getConfig().getBoolean(ExecConstants.IMPERSONATION_ENABLED);

try {
// TODO: DRILL-6412. Clients for plugin should be instantiated only for the case, when plugin is enabled
processUserMetastoreClient =
DrillHiveMetaStoreClient.createCloseableClientWithCaching(hiveConf);
} catch (MetaException e) {
@@ -82,12 +82,9 @@ public HiveSchemaFactory(final HiveStoragePlugin plugin, final String name, fina
.newBuilder()
.expireAfterAccess(10, TimeUnit.MINUTES)
.maximumSize(5) // Up to 5 clients for impersonation-enabled.
.removalListener(new RemovalListener<String, DrillHiveMetaStoreClient>() {
@Override
public void onRemoval(RemovalNotification<String, DrillHiveMetaStoreClient> notification) {
DrillHiveMetaStoreClient client = notification.getValue();
client.close();
}
.removalListener((RemovalListener<String, DrillHiveMetaStoreClient>) notification -> {
DrillHiveMetaStoreClient client = notification.getValue();
client.close();
})
.build(new CacheLoader<String, DrillHiveMetaStoreClient>() {
@Override
@@ -2,14 +2,16 @@
"storage":{
hive : {
type:"hive",
enabled: false,
config : {
"hive.metastore.uris" : "",
"javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../sample-data/drill_hive_db;create=true",
"hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh",
"fs.default.name" : "file:///",
"hive.metastore.sasl.enabled" : "false"
}
"hive.metastore.sasl.enabled" : "false",
"hive.metastore.schema.verification": "false",
"datanucleus.schema.autoCreateAll": "true"
},
enabled: false
}
}
}
@@ -2,15 +2,16 @@
"storage" : {
derby : {
type : "jdbc",
enabled : true,
driver : "org.apache.derby.jdbc.ClientDriver",
url : "jdbc:derby://localhost:${derby.reserved.port}/memory:${derby.database.name};user=root;password=root"
url : "jdbc:derby://localhost:${derby.reserved.port}/memory:${derby.database.name};user=root;password=root",
enabled : true
},
mysql : {
type : "jdbc",
enabled : true,
driver : "com.mysql.jdbc.Driver",
url : "jdbc:mysql://localhost:${mysql.reserved.port}/${mysql.database.name}?user=root&password=root&useJDBCCompliantTimezoneShift=true"
url : "jdbc:mysql://localhost:${mysql.reserved.port}/${mysql.database.name}?user=root&password=root&useJDBCCompliantTimezoneShift=true",
enabled : true
}
}
}
@@ -2,8 +2,8 @@
"storage":{
kafka : {
type:"kafka",
enabled: false,
kafkaConsumerProps: {"bootstrap.servers":"localhost:9092", "group.id" : "drill-consumer"}
kafkaConsumerProps: {"bootstrap.servers":"localhost:9092", "group.id" : "drill-consumer"},
enabled: false
}
}
}
@@ -2,8 +2,8 @@
"storage":{
mongo : {
type:"mongo",
enabled: false,
connection:"mongodb://localhost:27017/"
connection:"mongodb://localhost:27017/",
enabled: false
}
}
}
@@ -420,6 +420,11 @@
<source>src/resources/drill-on-yarn-example.conf</source>
<outputDirectory>conf</outputDirectory>
<fileMode>0640</fileMode>
</file>
</file>
<file>
<source>src/resources/storage-plugins-override-example.conf</source>
<outputDirectory>conf</outputDirectory>
<fileMode>0640</fileMode>
</file>
</files>
</assembly>
@@ -58,17 +58,10 @@ drill.exec: {
batch.size: 4000
},
partition.column.label: "dir"
}
},
metrics : {
context: "drillbit",
jmx: {
enabled : true
},
log: {
enabled : false,
interval : 60
}
# The action on the storage-plugins-override.conf after it's use.
# Possible values are "none" (default), "rename", "remove"
action_on_plugins_override_file: "none"
},
zk: {
connect: "localhost:2181",
@@ -252,6 +245,15 @@ drill.exec: {
#ssl provider. May be "JDK" or "OPENSSL". Default is "JDK"
provider: "JDK"
}
}

},

drill.metrics : {
context: "drillbit",
jmx: {
enabled : true
},
log: {
enabled : false,
interval : 60
}
}
@@ -0,0 +1,65 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file involves storage plugins configs, which can be updated on the Drill start-up.
# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.

"storage":{
cp: {
type: "file",
connection: "classpath:///",
formats: {
"csv" : {
type: "text",
extensions: [ "csv" ],
delimiter: ","
}
}
}
}
"storage":{
dfs: {
type: "file",
connection: "hdfs:///",
workspaces: {
"root": {
"location": "/",
"writable": false,
"defaultInputFormat": null,
"allowAccessOutsideWorkspace": false
}
},
formats: {
"parquet": {
"type": "parquet"
}
},
enabled: false
}
}
"storage":{
mongo : {
type:"mongo",
connection:"mongodb://test_host:27017/",
enabled: true
}
}
"storage": {
openTSDB: {
type: "openTSDB",
connection: "http://localhost:8888",
enabled: true
}
}
@@ -106,7 +106,6 @@
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.0.0</version>
</dependency>

<!-- Logging -->
@@ -205,6 +205,10 @@
<artifactId>jackson-module-afterburner</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.honton.chas.hocon</groupId>
<artifactId>jackson-dataformat-hocon</artifactId>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.ext</groupId>
<artifactId>jersey-mvc-freemarker</artifactId>

0 comments on commit 0ae7035

Please sign in to comment.