From 219b66cb832321d91ef689d5b6b16e2901cab900 Mon Sep 17 00:00:00 2001 From: Arina Ielchiieva Date: Wed, 30 Nov 2016 15:30:35 +0000 Subject: [PATCH] DRILL-5085: Add / update description for dynamic UDFs directories in drill-env.sh and drill-module.conf --- distribution/src/resources/drill-env.sh | 7 ++++- .../src/resources/drill-override-example.conf | 16 ++++++++++- .../org/apache/drill/exec/ExecConstants.java | 2 +- .../fn/FunctionImplementationRegistry.java | 15 +++++----- .../fn/registry/RemoteFunctionRegistry.java | 13 +++++---- .../src/main/resources/drill-module.conf | 28 +++++++++---------- .../apache/drill/TestDynamicUDFSupport.java | 1 + 7 files changed, 52 insertions(+), 30 deletions(-) diff --git a/distribution/src/resources/drill-env.sh b/distribution/src/resources/drill-env.sh index cd11b479993..62dffe65104 100644 --- a/distribution/src/resources/drill-env.sh +++ b/distribution/src/resources/drill-env.sh @@ -141,4 +141,9 @@ # Arguments passed to sqlline (the Drill shell) at all times: whether # Drill is embedded in Sqlline or not. -#export DRILL_SHELL_JAVA_OPTS="..." \ No newline at end of file +#export DRILL_SHELL_JAVA_OPTS="..." + +# Location Drill should use for temporary files, such as downloaded dynamic UDFs jars. +# Set to "/tmp" by default. +# +# export DRILL_TMP_DIR="..." diff --git a/distribution/src/resources/drill-override-example.conf b/distribution/src/resources/drill-override-example.conf index 4be4aa2c487..f9d27b3b4ec 100644 --- a/distribution/src/resources/drill-override-example.conf +++ b/distribution/src/resources/drill-override-example.conf @@ -170,7 +170,21 @@ drill.exec: { threadpool_size: 8, decode_threadpool_size: 1 }, - debug.error_on_leak: true + debug.error_on_leak: true, + # Settings for Dynamic UDFs. + # See https://gist.github.com/arina-ielchiieva/a1c4cfa3890145c5ecb1b70a39cbff55#file-dynamicudfssupport-md. + udf: { + # number of retry attempts to update remote function registry + # if registry version was changed during update + retry-attempts: 10, + directory: { + # Override this property if custom file system should be used to create remote directories + # instead of default taken from Hadoop configuration + fs: "hdfs:///", + # Set this property if custom absolute root should be used for remote directories + root: "/app/drill" + } + } } # Below SSL parameters need to be set for custom transport layer settings. diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index bb451c92f42..0eb9c586bd7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -116,9 +116,9 @@ public interface ExecConstants { * Configuration properties connected with dynamic UDFs support */ String UDF_RETRY_ATTEMPTS = "drill.exec.udf.retry-attempts"; + String UDF_DIRECTORY_LOCAL = "drill.exec.udf.directory.local"; String UDF_DIRECTORY_FS = "drill.exec.udf.directory.fs"; String UDF_DIRECTORY_ROOT = "drill.exec.udf.directory.root"; - String UDF_DIRECTORY_LOCAL = "drill.exec.udf.directory.local"; String UDF_DIRECTORY_STAGING = "drill.exec.udf.directory.staging"; String UDF_DIRECTORY_REGISTRY = "drill.exec.udf.directory.registry"; String UDF_DIRECTORY_TMP = "drill.exec.udf.directory.tmp"; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java index d0aa33f12fa..590a99a5cb6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java @@ -377,7 +377,6 @@ private List getMissingJars(RemoteFunctionRegistry remoteFunctionRegistr * Creates local udf directory, if it doesn't exist. * Checks if local udf directory is a directory and if current application has write rights on it. * Attempts to clean up local udf directory in case jars were left after previous drillbit run. - * Local udf directory path is concatenated from drill temporary directory and ${drill.exec.udf.directory.local}. * * @param config drill config * @return path to local udf directory @@ -395,22 +394,24 @@ private Path getLocalUdfDir(DrillConfig config) { } catch (IOException e) { throw new DrillRuntimeException("Error during local udf directory clean up", e); } + logger.info("Created and validated local udf directory [{}]", udfPath); return new Path(udfDir.toURI()); } /** - * First tries to get drill temporary directory value from environmental variable $DRILL_TMP_DIR, - * then from config ${drill.tmp-dir}. + * First tries to get drill temporary directory value from from config ${drill.tmp-dir}, + * then checks environmental variable $DRILL_TMP_DIR. * If value is still missing, generates directory using {@link Files#createTempDir()}. * If temporary directory was generated, sets {@link #deleteTmpDir} to true * to delete directory on drillbit exit. * @return drill temporary directory path */ private File getTmpDir(DrillConfig config) { - String drillTempDir = System.getenv("DRILL_TMP_DIR"); - - if (drillTempDir == null && config.hasPath(ExecConstants.DRILL_TMP_DIR)) { + String drillTempDir; + if (config.hasPath(ExecConstants.DRILL_TMP_DIR)) { drillTempDir = config.getString(ExecConstants.DRILL_TMP_DIR); + } else { + drillTempDir = System.getenv("DRILL_TMP_DIR"); } if (drillTempDir == null) { @@ -477,7 +478,7 @@ public void close() { * Will unregister all functions associated with the jar name * and delete binary and source associated with the jar from local udf directory */ - public class UnregistrationListener implements TransientStoreListener { + private class UnregistrationListener implements TransientStoreListener { @Override public void onChange(TransientStoreEvent event) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java index 4ce4a192f05..e5e62eba64f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/registry/RemoteFunctionRegistry.java @@ -81,8 +81,7 @@ */ public class RemoteFunctionRegistry implements AutoCloseable { - public static final String REGISTRY = "registry"; - + private static final String registry_path = "registry"; private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RemoteFunctionRegistry.class); private static final ObjectMapper mapper = new ObjectMapper().enable(INDENT_OUTPUT); @@ -108,15 +107,15 @@ public void init(DrillConfig config, PersistentStoreProvider storeProvider, Clus } public Registry getRegistry() { - return registry.get(REGISTRY, null); + return registry.get(registry_path, null); } public Registry getRegistry(DataChangeVersion version) { - return registry.get(REGISTRY, version); + return registry.get(registry_path, version); } public void updateRegistry(Registry registryContent, DataChangeVersion version) throws VersionMismatchException { - registry.put(REGISTRY, registryContent, version); + registry.put(registry_path, registryContent, version); } public void submitForUnregistration(String jar) { @@ -168,7 +167,7 @@ private void prepareStores(PersistentStoreProvider storeProvider, ClusterCoordin .persist() .build(); registry = storeProvider.getOrCreateStore(registrationConfig); - registry.putIfAbsent(REGISTRY, Registry.getDefaultInstance()); + registry.putIfAbsent(registry_path, Registry.getDefaultInstance()); } catch (StoreException e) { throw new DrillRuntimeException("Failure while loading remote registry.", e); } @@ -189,6 +188,7 @@ private void prepareStores(PersistentStoreProvider storeProvider, ClusterCoordin * if not set, uses user home directory instead. */ private void prepareAreas(DrillConfig config) { + logger.info("Preparing three remote udf areas: staging, registry and tmp."); Configuration conf = new Configuration(); if (config.hasPath(ExecConstants.UDF_DIRECTORY_FS)) { conf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.getString(ExecConstants.UDF_DIRECTORY_FS)); @@ -245,6 +245,7 @@ private Path createArea(FileSystem fs, String root, String directory) { } catch (Exception e) { DrillRuntimeException.format(e, "Error during udf area creation [%s] on file system [%s]", fullPath, fs.getUri()); } + logger.info("Created remote udf area [{}] on file system [{}]", fullPath, fs.getUri()); return path; } diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index deb31b34e88..ab73cec8a70 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -45,11 +45,10 @@ drill.client: { supports-complex-types: true } -// Directory is used as base for temporary storage of Dynamic UDF jars. -// Set this property if you want to have custom temporary directory, instead of generated at runtime. -// By default ${DRILL_TMP_DIR} is used if set. -// drill.tmp-dir: "/tmp" -// drill.tmp-dir: ${?DRILL_TMP_DIR} +// Location Drill uses for temporary files, such as downloaded dynamic UDFs jars. +// By default ${DRILL_TMP_DIR} is used if set or ${drill.tmp-dir} if it's been overridden. +drill.tmp-dir: "/tmp" +drill.tmp-dir: ${?DRILL_TMP_DIR} drill.exec: { cluster-id: "drillbits1" @@ -202,24 +201,25 @@ drill.exec: { udf: { retry-attempts: 5, directory: { + // Base directory for remote and local udf directories, unique among clusters. + base: ${drill.exec.zk.root}"/udf", + + // Path to local udf directory, always created on local file system. + // Root for these directory is generated at runtime unless Drill temporary directory is set. + local: ${drill.exec.udf.directory.base}"/udf/local", + // Set this property if custom file system should be used to create remote directories, ex: fs: "file:///". // fs: "", // Set this property if custom absolute root should be used for remote directories, ex: root: "/app/drill". // root: "", - // Relative base directory for all udf directories (local and remote). - base: ${drill.exec.zk.root}"/udf", - - // Relative path to local udf directory. - // Generated temporary directory is used as root unless ${DRILL_TMP_DIR} or ${drill.tmp-dir} is set. - local: ${drill.exec.udf.directory.base}"/local" - // Relative path to all remote udf directories. - // Directories are created under default file system unless ${drill.exec.udf.directory.fs} is set. + // Directories are created under default file system taken from Hadoop configuration + // unless ${drill.exec.udf.directory.fs} is set. // User home directory is used as root unless ${drill.exec.udf.directory.root} is set. staging: ${drill.exec.udf.directory.base}"/staging", registry: ${drill.exec.udf.directory.base}"/registry", - tmp: ${drill.exec.udf.directory.base}"/tmp", + tmp: ${drill.exec.udf.directory.base}"/tmp" } } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java index 4676e393b9c..78cdbe2c187 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java @@ -78,6 +78,7 @@ public class TestDynamicUDFSupport extends BaseTestQuery { public void setup() { Properties overrideProps = new Properties(); overrideProps.setProperty("drill.exec.udf.directory.root", base.getRoot().getPath()); + overrideProps.setProperty("drill.tmp-dir", base.getRoot().getPath()); updateTestCluster(1, DrillConfig.create(overrideProps)); }