From f156dfa8946899a0f98f7227a91246c16e67ceb7 Mon Sep 17 00:00:00 2001 From: Jyothsna Donapati Date: Wed, 7 Jun 2017 11:15:08 -0700 Subject: [PATCH] DRILL-5547:Linking config options with system option manager --- .../org/apache/drill/exec/ExecConstants.java | 173 ++++++++------- .../exec/compile/ClassCompilerSelector.java | 6 +- .../planner/fragment/SimpleParallelizer.java | 5 +- .../planner/physical/PlannerSettings.java | 74 +++---- .../sql/handlers/DefaultSqlHandler.java | 4 +- .../sql/handlers/SetOptionHandler.java | 21 +- .../rpc/user/InboundImpersonationManager.java | 4 +- .../drill/exec/rpc/user/UserSession.java | 6 +- .../apache/drill/exec/server/Drillbit.java | 3 +- .../drill/exec/server/DrillbitContext.java | 2 +- .../server/options/BaseOptionManager.java | 1 + .../server/options/DrillConfigIterator.java | 7 +- .../server/options/FallbackOptionManager.java | 27 ++- .../exec/server/options/OptionValidator.java | 10 +- .../exec/server/options/OptionValue.java | 46 ++-- .../server/options/SessionOptionManager.java | 10 +- .../server/options/SystemOptionManager.java | 97 +++++--- .../exec/server/options/TypeValidators.java | 131 ++++++++--- .../exec/server/rest/StatusResources.java | 2 +- .../store/sys/ExtendedOptionIterator.java | 155 +++++++++++++ .../drill/exec/store/sys/OptionIterator.java | 7 +- .../drill/exec/store/sys/SystemTable.java | 8 + .../drill/exec/testing/ExecutionControls.java | 14 +- .../exec/util/MemoryAllocationUtilities.java | 4 +- .../src/main/resources/drill-module.conf | 147 ++++++++++++ .../java/org/apache/drill/PlanningBase.java | 2 +- .../java/org/apache/drill/QueryTestUtil.java | 3 +- .../java/org/apache/drill/exec/ExecTest.java | 2 +- .../exec/compile/CodeCompilerTestFactory.java | 2 +- .../exec/compile/TestClassTransformation.java | 13 +- .../compile/bytecode/ReplaceMethodInvoke.java | 2 +- .../TestInboundImpersonationPrivileges.java | 2 +- .../partitionsender/TestPartitionSender.java | 11 +- .../exec/testing/ControlsInjectionUtil.java | 3 +- .../work/metadata/TestMetadataProvider.java | 8 +- .../org/apache/drill/test/FixtureBuilder.java | 9 + .../apache/drill/test/OperatorFixture.java | 9 +- .../apache/drill/test/TestConfigLinkage.java | 209 ++++++++++++++++++ .../drill/jdbc/test/TestJdbcMetadata.java | 4 +- 39 files changed, 979 insertions(+), 264 deletions(-) create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java create mode 100644 exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index a88875f8338..95ee00e715c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -29,8 +29,8 @@ import org.apache.drill.exec.server.options.TypeValidators.RangeDoubleValidator; import org.apache.drill.exec.server.options.TypeValidators.RangeLongValidator; import org.apache.drill.exec.server.options.TypeValidators.StringValidator; +import org.apache.drill.exec.server.options.TypeValidators.MaxWidthValidator; import org.apache.drill.exec.testing.ExecutionControls; -import org.apache.drill.exec.util.ImpersonationUtil; public interface ExecConstants { String ZK_RETRY_TIMES = "drill.exec.zk.retry.count"; @@ -88,22 +88,22 @@ public interface ExecConstants { // External Sort Runtime options - BooleanValidator EXTERNAL_SORT_DISABLE_MANAGED_OPTION = new BooleanValidator("exec.sort.disable_managed", false); + BooleanValidator EXTERNAL_SORT_DISABLE_MANAGED_OPTION = new BooleanValidator("exec.sort.disable_managed"); // Hash Aggregate Options String HASHAGG_NUM_PARTITIONS = "drill.exec.hashagg.num_partitions"; String HASHAGG_NUM_PARTITIONS_KEY = "exec.hashagg.num_partitions"; - LongValidator HASHAGG_NUM_PARTITIONS_VALIDATOR = new RangeLongValidator(HASHAGG_NUM_PARTITIONS_KEY, 1, 128, 32); // 1 means - no spilling + LongValidator HASHAGG_NUM_PARTITIONS_VALIDATOR = new RangeLongValidator(HASHAGG_NUM_PARTITIONS_KEY, 1, 128); // 1 means - no spilling String HASHAGG_MAX_MEMORY = "drill.exec.hashagg.mem_limit"; String HASHAGG_MAX_MEMORY_KEY = "exec.hashagg.mem_limit"; - LongValidator HASHAGG_MAX_MEMORY_VALIDATOR = new RangeLongValidator(HASHAGG_MAX_MEMORY_KEY, 0, Integer.MAX_VALUE, 0); + LongValidator HASHAGG_MAX_MEMORY_VALIDATOR = new RangeLongValidator(HASHAGG_MAX_MEMORY_KEY, 0, Integer.MAX_VALUE); // min batches is used for tuning (each partition needs so many batches when planning the number of partitions, // or reserve this number when calculating whether the remaining available memory is too small and requires a spill.) // Low value may OOM (e.g., when incoming rows become wider), higher values use fewer partitions but are safer String HASHAGG_MIN_BATCHES_PER_PARTITION = "drill.exec.hashagg.min_batches_per_partition"; String HASHAGG_MIN_BATCHES_PER_PARTITION_KEY = "drill.exec.hashagg.min_batches_per_partition"; - LongValidator HASHAGG_MIN_BATCHES_PER_PARTITION_VALIDATOR = new RangeLongValidator(HASHAGG_MIN_BATCHES_PER_PARTITION_KEY, 2, 5, 3); + LongValidator HASHAGG_MIN_BATCHES_PER_PARTITION_VALIDATOR = new RangeLongValidator(HASHAGG_MIN_BATCHES_PER_PARTITION_KEY, 2, 5); String HASHAGG_SPILL_DIRS = "drill.exec.hashagg.spill.directories"; String HASHAGG_SPILL_FILESYSTEM = "drill.exec.hashagg.spill.fs"; @@ -186,70 +186,70 @@ public interface ExecConstants { String DEFAULT_TEMPORARY_WORKSPACE = "drill.exec.default_temporary_workspace"; String OUTPUT_FORMAT_OPTION = "store.format"; - OptionValidator OUTPUT_FORMAT_VALIDATOR = new StringValidator(OUTPUT_FORMAT_OPTION, "parquet"); + OptionValidator OUTPUT_FORMAT_VALIDATOR = new StringValidator(OUTPUT_FORMAT_OPTION); String PARQUET_BLOCK_SIZE = "store.parquet.block-size"; String PARQUET_WRITER_USE_SINGLE_FS_BLOCK = "store.parquet.writer.use_single_fs_block"; OptionValidator PARQUET_WRITER_USE_SINGLE_FS_BLOCK_VALIDATOR = new BooleanValidator( - PARQUET_WRITER_USE_SINGLE_FS_BLOCK, false); - OptionValidator PARQUET_BLOCK_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_BLOCK_SIZE, Integer.MAX_VALUE, 512 * 1024 * 1024); + PARQUET_WRITER_USE_SINGLE_FS_BLOCK); + OptionValidator PARQUET_BLOCK_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_BLOCK_SIZE, Integer.MAX_VALUE); String PARQUET_PAGE_SIZE = "store.parquet.page-size"; - OptionValidator PARQUET_PAGE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_PAGE_SIZE, Integer.MAX_VALUE, 1024 * 1024); + OptionValidator PARQUET_PAGE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_PAGE_SIZE, Integer.MAX_VALUE); String PARQUET_DICT_PAGE_SIZE = "store.parquet.dictionary.page-size"; - OptionValidator PARQUET_DICT_PAGE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_DICT_PAGE_SIZE, Integer.MAX_VALUE, 1024 * 1024); + OptionValidator PARQUET_DICT_PAGE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_DICT_PAGE_SIZE, Integer.MAX_VALUE); String PARQUET_WRITER_COMPRESSION_TYPE = "store.parquet.compression"; OptionValidator PARQUET_WRITER_COMPRESSION_TYPE_VALIDATOR = new EnumeratedStringValidator( - PARQUET_WRITER_COMPRESSION_TYPE, "snappy", "snappy", "gzip", "none"); + PARQUET_WRITER_COMPRESSION_TYPE, "snappy", "gzip", "none"); String PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING = "store.parquet.enable_dictionary_encoding"; OptionValidator PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING_VALIDATOR = new BooleanValidator( - PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, false); + PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING); String PARQUET_VECTOR_FILL_THRESHOLD = "store.parquet.vector_fill_threshold"; - OptionValidator PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_THRESHOLD, 99l, 85l); + OptionValidator PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_THRESHOLD, 99l); String PARQUET_VECTOR_FILL_CHECK_THRESHOLD = "store.parquet.vector_fill_check_threshold"; - OptionValidator PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_CHECK_THRESHOLD, 100l, 10l); + OptionValidator PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_CHECK_THRESHOLD, 100l); String PARQUET_NEW_RECORD_READER = "store.parquet.use_new_reader"; - OptionValidator PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR = new BooleanValidator(PARQUET_NEW_RECORD_READER, false); + OptionValidator PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR = new BooleanValidator(PARQUET_NEW_RECORD_READER); String PARQUET_READER_INT96_AS_TIMESTAMP = "store.parquet.reader.int96_as_timestamp"; - OptionValidator PARQUET_READER_INT96_AS_TIMESTAMP_VALIDATOR = new BooleanValidator(PARQUET_READER_INT96_AS_TIMESTAMP, false); + OptionValidator PARQUET_READER_INT96_AS_TIMESTAMP_VALIDATOR = new BooleanValidator(PARQUET_READER_INT96_AS_TIMESTAMP); String PARQUET_PAGEREADER_ASYNC = "store.parquet.reader.pagereader.async"; - OptionValidator PARQUET_PAGEREADER_ASYNC_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_ASYNC, true); + OptionValidator PARQUET_PAGEREADER_ASYNC_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_ASYNC); // Number of pages the Async Parquet page reader will read before blocking String PARQUET_PAGEREADER_QUEUE_SIZE = "store.parquet.reader.pagereader.queuesize"; - OptionValidator PARQUET_PAGEREADER_QUEUE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_PAGEREADER_QUEUE_SIZE, Integer.MAX_VALUE, 2); + OptionValidator PARQUET_PAGEREADER_QUEUE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_PAGEREADER_QUEUE_SIZE, Integer.MAX_VALUE); String PARQUET_PAGEREADER_ENFORCETOTALSIZE = "store.parquet.reader.pagereader.enforceTotalSize"; - OptionValidator PARQUET_PAGEREADER_ENFORCETOTALSIZE_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_ENFORCETOTALSIZE, false); + OptionValidator PARQUET_PAGEREADER_ENFORCETOTALSIZE_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_ENFORCETOTALSIZE); String PARQUET_COLUMNREADER_ASYNC = "store.parquet.reader.columnreader.async"; - OptionValidator PARQUET_COLUMNREADER_ASYNC_VALIDATOR = new BooleanValidator(PARQUET_COLUMNREADER_ASYNC, false); + OptionValidator PARQUET_COLUMNREADER_ASYNC_VALIDATOR = new BooleanValidator(PARQUET_COLUMNREADER_ASYNC); // Use a buffering reader for parquet page reader String PARQUET_PAGEREADER_USE_BUFFERED_READ = "store.parquet.reader.pagereader.bufferedread"; - OptionValidator PARQUET_PAGEREADER_USE_BUFFERED_READ_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_USE_BUFFERED_READ, true); + OptionValidator PARQUET_PAGEREADER_USE_BUFFERED_READ_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_USE_BUFFERED_READ); // Size in MiB of the buffer the Parquet page reader will use to read from disk. Default is 1 MiB String PARQUET_PAGEREADER_BUFFER_SIZE = "store.parquet.reader.pagereader.buffersize"; - OptionValidator PARQUET_PAGEREADER_BUFFER_SIZE_VALIDATOR = new LongValidator(PARQUET_PAGEREADER_BUFFER_SIZE, 1*1024*1024); + OptionValidator PARQUET_PAGEREADER_BUFFER_SIZE_VALIDATOR = new LongValidator(PARQUET_PAGEREADER_BUFFER_SIZE); // try to use fadvise if available String PARQUET_PAGEREADER_USE_FADVISE = "store.parquet.reader.pagereader.usefadvise"; - OptionValidator PARQUET_PAGEREADER_USE_FADVISE_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_USE_FADVISE, false); + OptionValidator PARQUET_PAGEREADER_USE_FADVISE_VALIDATOR = new BooleanValidator(PARQUET_PAGEREADER_USE_FADVISE); - OptionValidator COMPILE_SCALAR_REPLACEMENT = new BooleanValidator("exec.compile.scalar_replacement", false); + OptionValidator COMPILE_SCALAR_REPLACEMENT = new BooleanValidator("exec.compile.scalar_replacement"); String JSON_ALL_TEXT_MODE = "store.json.all_text_mode"; - BooleanValidator JSON_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(JSON_ALL_TEXT_MODE, false); - BooleanValidator JSON_EXTENDED_TYPES = new BooleanValidator("store.json.extended_types", false); - BooleanValidator JSON_WRITER_UGLIFY = new BooleanValidator("store.json.writer.uglify", false); - BooleanValidator JSON_WRITER_SKIPNULLFIELDS = new BooleanValidator("store.json.writer.skip_null_fields", true); + BooleanValidator JSON_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(JSON_ALL_TEXT_MODE); + BooleanValidator JSON_EXTENDED_TYPES = new BooleanValidator("store.json.extended_types"); + BooleanValidator JSON_WRITER_UGLIFY = new BooleanValidator("store.json.writer.uglify"); + BooleanValidator JSON_WRITER_SKIPNULLFIELDS = new BooleanValidator("store.json.writer.skip_null_fields"); String JSON_READER_SKIP_INVALID_RECORDS_FLAG = "store.json.reader.skip_invalid_records"; - BooleanValidator JSON_SKIP_MALFORMED_RECORDS_VALIDATOR = new BooleanValidator(JSON_READER_SKIP_INVALID_RECORDS_FLAG, false); + BooleanValidator JSON_SKIP_MALFORMED_RECORDS_VALIDATOR = new BooleanValidator(JSON_READER_SKIP_INVALID_RECORDS_FLAG); String JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG = "store.json.reader.print_skipped_invalid_record_number"; - BooleanValidator JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG_VALIDATOR = new BooleanValidator(JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG, false); + BooleanValidator JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG_VALIDATOR = new BooleanValidator(JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG); DoubleValidator TEXT_ESTIMATED_ROW_SIZE = new RangeDoubleValidator( - "store.text.estimated_row_size_bytes", 1, Long.MAX_VALUE, 100.0); + "store.text.estimated_row_size_bytes", 1, Long.MAX_VALUE); /** * The column label (for directory levels) in results when querying files in a directory @@ -259,59 +259,61 @@ public interface ExecConstants { * |- baz - b.parquet */ String FILESYSTEM_PARTITION_COLUMN_LABEL = "drill.exec.storage.file.partition.column.label"; - OptionValidator FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR = new StringValidator(FILESYSTEM_PARTITION_COLUMN_LABEL, "dir"); + OptionValidator FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR = new StringValidator(FILESYSTEM_PARTITION_COLUMN_LABEL); /** * Implicit file columns */ String IMPLICIT_FILENAME_COLUMN_LABEL = "drill.exec.storage.implicit.filename.column.label"; - OptionValidator IMPLICIT_FILENAME_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FILENAME_COLUMN_LABEL, "filename"); + OptionValidator IMPLICIT_FILENAME_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FILENAME_COLUMN_LABEL); String IMPLICIT_SUFFIX_COLUMN_LABEL = "drill.exec.storage.implicit.suffix.column.label"; - OptionValidator IMPLICIT_SUFFIX_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_SUFFIX_COLUMN_LABEL, "suffix"); + OptionValidator IMPLICIT_SUFFIX_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_SUFFIX_COLUMN_LABEL); String IMPLICIT_FQN_COLUMN_LABEL = "drill.exec.storage.implicit.fqn.column.label"; - OptionValidator IMPLICIT_FQN_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FQN_COLUMN_LABEL, "fqn"); + OptionValidator IMPLICIT_FQN_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FQN_COLUMN_LABEL); String IMPLICIT_FILEPATH_COLUMN_LABEL = "drill.exec.storage.implicit.filepath.column.label"; - OptionValidator IMPLICIT_FILEPATH_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FILEPATH_COLUMN_LABEL, "filepath"); + OptionValidator IMPLICIT_FILEPATH_COLUMN_LABEL_VALIDATOR = new StringValidator(IMPLICIT_FILEPATH_COLUMN_LABEL); String JSON_READ_NUMBERS_AS_DOUBLE = "store.json.read_numbers_as_double"; - BooleanValidator JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(JSON_READ_NUMBERS_AS_DOUBLE, false); + BooleanValidator JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(JSON_READ_NUMBERS_AS_DOUBLE); String MONGO_ALL_TEXT_MODE = "store.mongo.all_text_mode"; - OptionValidator MONGO_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(MONGO_ALL_TEXT_MODE, false); + OptionValidator MONGO_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(MONGO_ALL_TEXT_MODE); String MONGO_READER_READ_NUMBERS_AS_DOUBLE = "store.mongo.read_numbers_as_double"; - OptionValidator MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(MONGO_READER_READ_NUMBERS_AS_DOUBLE, false); + OptionValidator MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR = new BooleanValidator(MONGO_READER_READ_NUMBERS_AS_DOUBLE); String MONGO_BSON_RECORD_READER = "store.mongo.bson.record.reader"; - OptionValidator MONGO_BSON_RECORD_READER_VALIDATOR = new BooleanValidator(MONGO_BSON_RECORD_READER, true); + OptionValidator MONGO_BSON_RECORD_READER_VALIDATOR = new BooleanValidator(MONGO_BSON_RECORD_READER); - BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type", false); + BooleanValidator ENABLE_UNION_TYPE = new BooleanValidator("exec.enable_union_type"); // TODO: We need to add a feature that enables storage plugins to add their own options. Currently we have to declare // in core which is not right. Move this option and above two mongo plugin related options once we have the feature. String HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS = "store.hive.optimize_scan_with_native_readers"; OptionValidator HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS_VALIDATOR = - new BooleanValidator(HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS, false); + new BooleanValidator(HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS); String SLICE_TARGET = "planner.slice_target"; long SLICE_TARGET_DEFAULT = 100000l; - PositiveLongValidator SLICE_TARGET_OPTION = new PositiveLongValidator(SLICE_TARGET, Long.MAX_VALUE, - SLICE_TARGET_DEFAULT); + PositiveLongValidator SLICE_TARGET_OPTION = new PositiveLongValidator(SLICE_TARGET, Long.MAX_VALUE + ); String CAST_TO_NULLABLE_NUMERIC = "drill.exec.functions.cast_empty_string_to_null"; - BooleanValidator CAST_TO_NULLABLE_NUMERIC_OPTION = new BooleanValidator(CAST_TO_NULLABLE_NUMERIC, false); + BooleanValidator CAST_TO_NULLABLE_NUMERIC_OPTION = new BooleanValidator(CAST_TO_NULLABLE_NUMERIC); /** * HashTable runtime settings */ String MIN_HASH_TABLE_SIZE_KEY = "exec.min_hash_table_size"; - PositiveLongValidator MIN_HASH_TABLE_SIZE = new PositiveLongValidator(MIN_HASH_TABLE_SIZE_KEY, HashTable.MAXIMUM_CAPACITY, HashTable.DEFAULT_INITIAL_CAPACITY); + PositiveLongValidator MIN_HASH_TABLE_SIZE = new PositiveLongValidator(MIN_HASH_TABLE_SIZE_KEY, HashTable.MAXIMUM_CAPACITY); String MAX_HASH_TABLE_SIZE_KEY = "exec.max_hash_table_size"; - PositiveLongValidator MAX_HASH_TABLE_SIZE = new PositiveLongValidator(MAX_HASH_TABLE_SIZE_KEY, HashTable.MAXIMUM_CAPACITY, HashTable.MAXIMUM_CAPACITY); + PositiveLongValidator MAX_HASH_TABLE_SIZE = new PositiveLongValidator(MAX_HASH_TABLE_SIZE_KEY, HashTable.MAXIMUM_CAPACITY); /** * Limits the maximum level of parallelization to this factor time the number of Drillbits */ + String CPU_LOAD_AVERAGE_KEY = "planner.cpu_load_average"; + DoubleValidator CPU_LOAD_AVERAGE = new DoubleValidator(CPU_LOAD_AVERAGE_KEY); String MAX_WIDTH_PER_NODE_KEY = "planner.width.max_per_node"; - OptionValidator MAX_WIDTH_PER_NODE = new PositiveLongValidator(MAX_WIDTH_PER_NODE_KEY, Integer.MAX_VALUE, (long) Math.ceil(Runtime.getRuntime().availableProcessors() * 0.70)); + MaxWidthValidator MAX_WIDTH_PER_NODE = new MaxWidthValidator(MAX_WIDTH_PER_NODE_KEY); /** * The maximum level or parallelization any stage of the query can do. Note that while this @@ -319,19 +321,19 @@ public interface ExecConstants { * number of we want to do things like speed results return. */ String MAX_WIDTH_GLOBAL_KEY = "planner.width.max_per_query"; - OptionValidator MAX_WIDTH_GLOBAL = new PositiveLongValidator(MAX_WIDTH_GLOBAL_KEY, Integer.MAX_VALUE, 1000); + OptionValidator MAX_WIDTH_GLOBAL = new PositiveLongValidator(MAX_WIDTH_GLOBAL_KEY, Integer.MAX_VALUE); /** * Factor by which a node with endpoint affinity will be favored while creating assignment */ String AFFINITY_FACTOR_KEY = "planner.affinity_factor"; - OptionValidator AFFINITY_FACTOR = new DoubleValidator(AFFINITY_FACTOR_KEY, 1.2d); + OptionValidator AFFINITY_FACTOR = new DoubleValidator(AFFINITY_FACTOR_KEY); String EARLY_LIMIT0_OPT_KEY = "planner.enable_limit0_optimization"; - BooleanValidator EARLY_LIMIT0_OPT = new BooleanValidator(EARLY_LIMIT0_OPT_KEY, false); + BooleanValidator EARLY_LIMIT0_OPT = new BooleanValidator(EARLY_LIMIT0_OPT_KEY); String ENABLE_MEMORY_ESTIMATION_KEY = "planner.memory.enable_memory_estimation"; - OptionValidator ENABLE_MEMORY_ESTIMATION = new BooleanValidator(ENABLE_MEMORY_ESTIMATION_KEY, false); + OptionValidator ENABLE_MEMORY_ESTIMATION = new BooleanValidator(ENABLE_MEMORY_ESTIMATION_KEY); /** * Maximum query memory per node (in MB). Re-plan with cheaper operators if memory estimation exceeds this limit. @@ -340,7 +342,7 @@ public interface ExecConstants { */ String MAX_QUERY_MEMORY_PER_NODE_KEY = "planner.memory.max_query_memory_per_node"; LongValidator MAX_QUERY_MEMORY_PER_NODE = new RangeLongValidator( - MAX_QUERY_MEMORY_PER_NODE_KEY, 1024 * 1024, Long.MAX_VALUE, 2 * 1024 * 1024 * 1024L); + MAX_QUERY_MEMORY_PER_NODE_KEY, 1024 * 1024, Long.MAX_VALUE); /** * Minimum memory alocated to each buffered operator instance. @@ -349,7 +351,7 @@ public interface ExecConstants { */ String MIN_MEMORY_PER_BUFFERED_OP_KEY = "planner.memory.min_memory_per_buffered_op"; LongValidator MIN_MEMORY_PER_BUFFERED_OP = new RangeLongValidator( - MIN_MEMORY_PER_BUFFERED_OP_KEY, 1024 * 1024, Long.MAX_VALUE, 40 * 1024 * 1024L); + MIN_MEMORY_PER_BUFFERED_OP_KEY, 1024 * 1024, Long.MAX_VALUE); /** * Extra query memory per node for non-blocking operators. @@ -360,51 +362,51 @@ public interface ExecConstants { */ String NON_BLOCKING_OPERATORS_MEMORY_KEY = "planner.memory.non_blocking_operators_memory"; OptionValidator NON_BLOCKING_OPERATORS_MEMORY = new PowerOfTwoLongValidator( - NON_BLOCKING_OPERATORS_MEMORY_KEY, 1 << 11, 1 << 6); + NON_BLOCKING_OPERATORS_MEMORY_KEY, 1 << 11); String HASH_JOIN_TABLE_FACTOR_KEY = "planner.memory.hash_join_table_factor"; - OptionValidator HASH_JOIN_TABLE_FACTOR = new DoubleValidator(HASH_JOIN_TABLE_FACTOR_KEY, 1.1d); + OptionValidator HASH_JOIN_TABLE_FACTOR = new DoubleValidator(HASH_JOIN_TABLE_FACTOR_KEY); String HASH_AGG_TABLE_FACTOR_KEY = "planner.memory.hash_agg_table_factor"; - OptionValidator HASH_AGG_TABLE_FACTOR = new DoubleValidator(HASH_AGG_TABLE_FACTOR_KEY, 1.1d); + OptionValidator HASH_AGG_TABLE_FACTOR = new DoubleValidator(HASH_AGG_TABLE_FACTOR_KEY); String AVERAGE_FIELD_WIDTH_KEY = "planner.memory.average_field_width"; - OptionValidator AVERAGE_FIELD_WIDTH = new PositiveLongValidator(AVERAGE_FIELD_WIDTH_KEY, Long.MAX_VALUE, 8); + OptionValidator AVERAGE_FIELD_WIDTH = new PositiveLongValidator(AVERAGE_FIELD_WIDTH_KEY, Long.MAX_VALUE); - BooleanValidator ENABLE_QUEUE = new BooleanValidator("exec.queue.enable", false); - LongValidator LARGE_QUEUE_SIZE = new PositiveLongValidator("exec.queue.large", 1000, 10); - LongValidator SMALL_QUEUE_SIZE = new PositiveLongValidator("exec.queue.small", 100000, 100); + BooleanValidator ENABLE_QUEUE = new BooleanValidator("exec.queue.enable"); + LongValidator LARGE_QUEUE_SIZE = new PositiveLongValidator("exec.queue.large", 1000); + LongValidator SMALL_QUEUE_SIZE = new PositiveLongValidator("exec.queue.small", 100000); LongValidator QUEUE_THRESHOLD_SIZE = new PositiveLongValidator("exec.queue.threshold", - Long.MAX_VALUE, 30000000); + Long.MAX_VALUE); LongValidator QUEUE_TIMEOUT = new PositiveLongValidator("exec.queue.timeout_millis", - Long.MAX_VALUE, 60 * 1000 * 5); + Long.MAX_VALUE); String ENABLE_VERBOSE_ERRORS_KEY = "exec.errors.verbose"; - OptionValidator ENABLE_VERBOSE_ERRORS = new BooleanValidator(ENABLE_VERBOSE_ERRORS_KEY, false); + OptionValidator ENABLE_VERBOSE_ERRORS = new BooleanValidator(ENABLE_VERBOSE_ERRORS_KEY); String ENABLE_NEW_TEXT_READER_KEY = "exec.storage.enable_new_text_reader"; - OptionValidator ENABLE_NEW_TEXT_READER = new BooleanValidator(ENABLE_NEW_TEXT_READER_KEY, true); + OptionValidator ENABLE_NEW_TEXT_READER = new BooleanValidator(ENABLE_NEW_TEXT_READER_KEY); String BOOTSTRAP_STORAGE_PLUGINS_FILE = "bootstrap-storage-plugins.json"; String DRILL_SYS_FILE_SUFFIX = ".sys.drill"; String ENABLE_WINDOW_FUNCTIONS = "window.enable"; - OptionValidator ENABLE_WINDOW_FUNCTIONS_VALIDATOR = new BooleanValidator(ENABLE_WINDOW_FUNCTIONS, true); + OptionValidator ENABLE_WINDOW_FUNCTIONS_VALIDATOR = new BooleanValidator(ENABLE_WINDOW_FUNCTIONS); String DRILLBIT_CONTROL_INJECTIONS = "drill.exec.testing.controls"; OptionValidator DRILLBIT_CONTROLS_VALIDATOR = - new ExecutionControls.ControlsOptionValidator(DRILLBIT_CONTROL_INJECTIONS, ExecutionControls.DEFAULT_CONTROLS, 1); + new ExecutionControls.ControlsOptionValidator(DRILLBIT_CONTROL_INJECTIONS, 1); String NEW_VIEW_DEFAULT_PERMS_KEY = "new_view_default_permissions"; OptionValidator NEW_VIEW_DEFAULT_PERMS_VALIDATOR = - new StringValidator(NEW_VIEW_DEFAULT_PERMS_KEY, "700"); + new StringValidator(NEW_VIEW_DEFAULT_PERMS_KEY); String CTAS_PARTITIONING_HASH_DISTRIBUTE = "store.partition.hash_distribute"; - BooleanValidator CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR = new BooleanValidator(CTAS_PARTITIONING_HASH_DISTRIBUTE, false); + BooleanValidator CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR = new BooleanValidator(CTAS_PARTITIONING_HASH_DISTRIBUTE); String ENABLE_BULK_LOAD_TABLE_LIST_KEY = "exec.enable_bulk_load_table_list"; - BooleanValidator ENABLE_BULK_LOAD_TABLE_LIST = new BooleanValidator(ENABLE_BULK_LOAD_TABLE_LIST_KEY, false); + BooleanValidator ENABLE_BULK_LOAD_TABLE_LIST = new BooleanValidator(ENABLE_BULK_LOAD_TABLE_LIST_KEY); /** * When getting Hive Table information with exec.enable_bulk_load_table_list set to true, @@ -412,20 +414,20 @@ public interface ExecConstants { * at a time. (The number of tables can get to be quite large.) */ String BULK_LOAD_TABLE_LIST_BULK_SIZE_KEY = "exec.bulk_load_table_list.bulk_size"; - PositiveLongValidator BULK_LOAD_TABLE_LIST_BULK_SIZE = new PositiveLongValidator(BULK_LOAD_TABLE_LIST_BULK_SIZE_KEY, Integer.MAX_VALUE, 1000); + PositiveLongValidator BULK_LOAD_TABLE_LIST_BULK_SIZE = new PositiveLongValidator(BULK_LOAD_TABLE_LIST_BULK_SIZE_KEY, Integer.MAX_VALUE); /** * Option whose value is a comma separated list of admin usernames. Admin users are users who have special privileges * such as changing system options. */ String ADMIN_USERS_KEY = "security.admin.users"; - StringValidator ADMIN_USERS_VALIDATOR = new StringValidator(ADMIN_USERS_KEY, ImpersonationUtil.getProcessUserName(), true); + StringValidator ADMIN_USERS_VALIDATOR = new StringValidator(ADMIN_USERS_KEY, true); /** * Option whose value is a comma separated list of admin usergroups. */ String ADMIN_USER_GROUPS_KEY = "security.admin.user_groups"; - StringValidator ADMIN_USER_GROUPS_VALIDATOR = new StringValidator(ADMIN_USER_GROUPS_KEY, "", true); + StringValidator ADMIN_USER_GROUPS_VALIDATOR = new StringValidator(ADMIN_USER_GROUPS_KEY, true); /** * Option whose value is a string representing list of inbound impersonation policies. * @@ -440,16 +442,17 @@ public interface ExecConstants { */ String IMPERSONATION_POLICIES_KEY = "exec.impersonation.inbound_policies"; StringValidator IMPERSONATION_POLICY_VALIDATOR = - new InboundImpersonationManager.InboundImpersonationPolicyValidator(IMPERSONATION_POLICIES_KEY, "[]"); + new InboundImpersonationManager.InboundImpersonationPolicyValidator(IMPERSONATION_POLICIES_KEY); + /** * Web settings */ String WEB_LOGS_MAX_LINES = "web.logs.max_lines"; - OptionValidator WEB_LOGS_MAX_LINES_VALIDATOR = new PositiveLongValidator(WEB_LOGS_MAX_LINES, Integer.MAX_VALUE, 10000); + OptionValidator WEB_LOGS_MAX_LINES_VALIDATOR = new PositiveLongValidator(WEB_LOGS_MAX_LINES, Integer.MAX_VALUE); String CODE_GEN_EXP_IN_METHOD_SIZE = "exec.java.compiler.exp_in_method_size"; - LongValidator CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR = new LongValidator(CODE_GEN_EXP_IN_METHOD_SIZE, 50); + LongValidator CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR = new LongValidator(CODE_GEN_EXP_IN_METHOD_SIZE); /** * Timeout for create prepare statement request. If the request exceeds this timeout, then request is timed out. @@ -457,10 +460,10 @@ public interface ExecConstants { */ String CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS = "prepare.statement.create_timeout_ms"; OptionValidator CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS_VALIDATOR = - new PositiveLongValidator(CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS, Integer.MAX_VALUE, 10000); + new PositiveLongValidator(CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS, Integer.MAX_VALUE); String DYNAMIC_UDF_SUPPORT_ENABLED = "exec.udf.enable_dynamic_support"; - BooleanValidator DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR = new BooleanValidator(DYNAMIC_UDF_SUPPORT_ENABLED, true, true); + BooleanValidator DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR = new BooleanValidator(DYNAMIC_UDF_SUPPORT_ENABLED); /** * Option to save query profiles. If false, no query profile will be saved @@ -468,7 +471,7 @@ public interface ExecConstants { */ String ENABLE_QUERY_PROFILE_OPTION = "exec.query_profile.save"; BooleanValidator ENABLE_QUERY_PROFILE_VALIDATOR = new BooleanValidator( - ENABLE_QUERY_PROFILE_OPTION, true, false); + ENABLE_QUERY_PROFILE_OPTION); /** * Profiles are normally written after the last client message to reduce latency. @@ -478,16 +481,16 @@ public interface ExecConstants { */ String QUERY_PROFILE_DEBUG_OPTION = "exec.query_profile.debug_mode"; BooleanValidator QUERY_PROFILE_DEBUG_VALIDATOR = new BooleanValidator( - QUERY_PROFILE_DEBUG_OPTION, false, false); + QUERY_PROFILE_DEBUG_OPTION); String USE_DYNAMIC_UDFS_KEY = "exec.udf.use_dynamic"; - BooleanValidator USE_DYNAMIC_UDFS = new BooleanValidator(USE_DYNAMIC_UDFS_KEY, true); + BooleanValidator USE_DYNAMIC_UDFS = new BooleanValidator(USE_DYNAMIC_UDFS_KEY); String QUERY_TRANSIENT_STATE_UPDATE_KEY = "exec.query.progress.update"; - BooleanValidator QUERY_TRANSIENT_STATE_UPDATE = new BooleanValidator(QUERY_TRANSIENT_STATE_UPDATE_KEY, true); + BooleanValidator QUERY_TRANSIENT_STATE_UPDATE = new BooleanValidator(QUERY_TRANSIENT_STATE_UPDATE_KEY); String PERSISTENT_TABLE_UMASK = "exec.persistent_table.umask"; - StringValidator PERSISTENT_TABLE_UMASK_VALIDATOR = new StringValidator(PERSISTENT_TABLE_UMASK, "002"); + StringValidator PERSISTENT_TABLE_UMASK_VALIDATOR = new StringValidator(PERSISTENT_TABLE_UMASK); /** * Enables batch iterator (operator) validation. Validation is normally enabled @@ -496,7 +499,7 @@ public interface ExecConstants { * a "production" Drill instance. */ String ENABLE_ITERATOR_VALIDATION_OPTION = "debug.validate_iterators"; - BooleanValidator ENABLE_ITERATOR_VALIDATOR = new BooleanValidator(ENABLE_ITERATOR_VALIDATION_OPTION, false); + BooleanValidator ENABLE_ITERATOR_VALIDATOR = new BooleanValidator(ENABLE_ITERATOR_VALIDATION_OPTION); /** * Boot-time config option to enable validation. Primarily used for tests. @@ -511,7 +514,7 @@ public interface ExecConstants { * each batch passed to each iterator. */ String ENABLE_VECTOR_VALIDATION_OPTION = "debug.validate_vectors"; - BooleanValidator ENABLE_VECTOR_VALIDATOR = new BooleanValidator(ENABLE_VECTOR_VALIDATION_OPTION, false); + BooleanValidator ENABLE_VECTOR_VALIDATOR = new BooleanValidator(ENABLE_VECTOR_VALIDATION_OPTION); /** * Boot-time config option to enable vector validation. Primarily used for diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompilerSelector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompilerSelector.java index 92b8430b9cc..9b6e86e84cc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompilerSelector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/ClassCompilerSelector.java @@ -75,12 +75,12 @@ public enum CompilerPolicy { public static final String JAVA_COMPILER_OPTION = "exec.java_compiler"; public static final String JAVA_COMPILER_JANINO_MAXSIZE_OPTION = "exec.java_compiler_janino_maxsize"; - public static final OptionValidator JAVA_COMPILER_JANINO_MAXSIZE = new LongValidator(JAVA_COMPILER_JANINO_MAXSIZE_OPTION, 256*1024); + public static final OptionValidator JAVA_COMPILER_JANINO_MAXSIZE = new LongValidator(JAVA_COMPILER_JANINO_MAXSIZE_OPTION); public static final String JAVA_COMPILER_DEBUG_OPTION = "exec.java_compiler_debug"; - public static final OptionValidator JAVA_COMPILER_DEBUG = new BooleanValidator(JAVA_COMPILER_DEBUG_OPTION, true); + public static final OptionValidator JAVA_COMPILER_DEBUG = new BooleanValidator(JAVA_COMPILER_DEBUG_OPTION); - public static final StringValidator JAVA_COMPILER_VALIDATOR = new StringValidator(JAVA_COMPILER_OPTION, CompilerPolicy.DEFAULT.toString()) { + public static final StringValidator JAVA_COMPILER_VALIDATOR = new StringValidator(JAVA_COMPILER_OPTION) { @Override public void validate(final OptionValue v, final OptionSet manager) { super.validate(v, manager); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java index c389e89cdef..4584bd5550a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java @@ -72,7 +72,10 @@ public SimpleParallelizer(QueryContext context) { OptionManager optionManager = context.getOptions(); long sliceTarget = optionManager.getOption(ExecConstants.SLICE_TARGET).num_val; this.parallelizationThreshold = sliceTarget > 0 ? sliceTarget : 1; - this.maxWidthPerNode = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val.intValue(); + double cpu_load_average = optionManager.getOption(ExecConstants.CPU_LOAD_AVERAGE); + final long maxWidth = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE); + // compute the maxwidth + this.maxWidthPerNode = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(cpu_load_average,maxWidth); this.maxGlobalWidth = optionManager.getOption(ExecConstants.MAX_WIDTH_GLOBAL_KEY).num_val.intValue(); this.affinityFactor = optionManager.getOption(ExecConstants.AFFINITY_FACTOR_KEY).float_val.intValue(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java index 15314bacbca..36285a50fae 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java @@ -51,42 +51,42 @@ public class PlannerSettings implements Context{ // max off heap memory for planning (16G) private static final long MAX_OFF_HEAP_ALLOCATION_IN_BYTES = 16l * 1024 * 1024 * 1024; - public static final OptionValidator CONSTANT_FOLDING = new BooleanValidator("planner.enable_constant_folding", true); - public static final OptionValidator EXCHANGE = new BooleanValidator("planner.disable_exchanges", false); - public static final OptionValidator HASHAGG = new BooleanValidator("planner.enable_hashagg", true); - public static final OptionValidator STREAMAGG = new BooleanValidator("planner.enable_streamagg", true); - public static final OptionValidator HASHJOIN = new BooleanValidator("planner.enable_hashjoin", true); - public static final OptionValidator MERGEJOIN = new BooleanValidator("planner.enable_mergejoin", true); - public static final OptionValidator NESTEDLOOPJOIN = new BooleanValidator("planner.enable_nestedloopjoin", true); - public static final OptionValidator MULTIPHASE = new BooleanValidator("planner.enable_multiphase_agg", true); - public static final OptionValidator BROADCAST = new BooleanValidator("planner.enable_broadcast_join", true); - public static final OptionValidator BROADCAST_THRESHOLD = new PositiveLongValidator("planner.broadcast_threshold", MAX_BROADCAST_THRESHOLD, 10000000); - public static final OptionValidator BROADCAST_FACTOR = new RangeDoubleValidator("planner.broadcast_factor", 0, Double.MAX_VALUE, 1.0d); - public static final OptionValidator NESTEDLOOPJOIN_FACTOR = new RangeDoubleValidator("planner.nestedloopjoin_factor", 0, Double.MAX_VALUE, 100.0d); - public static final OptionValidator NLJOIN_FOR_SCALAR = new BooleanValidator("planner.enable_nljoin_for_scalar_only", true); - public static final OptionValidator JOIN_ROW_COUNT_ESTIMATE_FACTOR = new RangeDoubleValidator("planner.join.row_count_estimate_factor", 0, Double.MAX_VALUE, 1.0d); - public static final OptionValidator MUX_EXCHANGE = new BooleanValidator("planner.enable_mux_exchange", true); - public static final OptionValidator DEMUX_EXCHANGE = new BooleanValidator("planner.enable_demux_exchange", false); - public static final OptionValidator PARTITION_SENDER_THREADS_FACTOR = new LongValidator("planner.partitioner_sender_threads_factor", 2); - public static final OptionValidator PARTITION_SENDER_MAX_THREADS = new LongValidator("planner.partitioner_sender_max_threads", 8); - public static final OptionValidator PARTITION_SENDER_SET_THREADS = new LongValidator("planner.partitioner_sender_set_threads", -1); - public static final OptionValidator PRODUCER_CONSUMER = new BooleanValidator("planner.add_producer_consumer", false); - public static final OptionValidator PRODUCER_CONSUMER_QUEUE_SIZE = new LongValidator("planner.producer_consumer_queue_size", 10); - public static final OptionValidator HASH_SINGLE_KEY = new BooleanValidator("planner.enable_hash_single_key", true); - public static final OptionValidator HASH_JOIN_SWAP = new BooleanValidator("planner.enable_hashjoin_swap", true); - public static final OptionValidator HASH_JOIN_SWAP_MARGIN_FACTOR = new RangeDoubleValidator("planner.join.hash_join_swap_margin_factor", 0, 100, 10d); + public static final OptionValidator CONSTANT_FOLDING = new BooleanValidator("planner.enable_constant_folding"); + public static final OptionValidator EXCHANGE = new BooleanValidator("planner.disable_exchanges"); + public static final OptionValidator HASHAGG = new BooleanValidator("planner.enable_hashagg"); + public static final OptionValidator STREAMAGG = new BooleanValidator("planner.enable_streamagg"); + public static final OptionValidator HASHJOIN = new BooleanValidator("planner.enable_hashjoin"); + public static final OptionValidator MERGEJOIN = new BooleanValidator("planner.enable_mergejoin"); + public static final OptionValidator NESTEDLOOPJOIN = new BooleanValidator("planner.enable_nestedloopjoin"); + public static final OptionValidator MULTIPHASE = new BooleanValidator("planner.enable_multiphase_agg"); + public static final OptionValidator BROADCAST = new BooleanValidator("planner.enable_broadcast_join"); + public static final OptionValidator BROADCAST_THRESHOLD = new PositiveLongValidator("planner.broadcast_threshold", MAX_BROADCAST_THRESHOLD); + public static final OptionValidator BROADCAST_FACTOR = new RangeDoubleValidator("planner.broadcast_factor", 0, Double.MAX_VALUE); + public static final OptionValidator NESTEDLOOPJOIN_FACTOR = new RangeDoubleValidator("planner.nestedloopjoin_factor", 0, Double.MAX_VALUE); + public static final OptionValidator NLJOIN_FOR_SCALAR = new BooleanValidator("planner.enable_nljoin_for_scalar_only"); + public static final OptionValidator JOIN_ROW_COUNT_ESTIMATE_FACTOR = new RangeDoubleValidator("planner.join.row_count_estimate_factor", 0, Double.MAX_VALUE); + public static final OptionValidator MUX_EXCHANGE = new BooleanValidator("planner.enable_mux_exchange"); + public static final OptionValidator DEMUX_EXCHANGE = new BooleanValidator("planner.enable_demux_exchange"); + public static final OptionValidator PARTITION_SENDER_THREADS_FACTOR = new LongValidator("planner.partitioner_sender_threads_factor"); + public static final OptionValidator PARTITION_SENDER_MAX_THREADS = new LongValidator("planner.partitioner_sender_max_threads"); + public static final OptionValidator PARTITION_SENDER_SET_THREADS = new LongValidator("planner.partitioner_sender_set_threads"); + public static final OptionValidator PRODUCER_CONSUMER = new BooleanValidator("planner.add_producer_consumer"); + public static final OptionValidator PRODUCER_CONSUMER_QUEUE_SIZE = new LongValidator("planner.producer_consumer_queue_size"); + public static final OptionValidator HASH_SINGLE_KEY = new BooleanValidator("planner.enable_hash_single_key"); + public static final OptionValidator HASH_JOIN_SWAP = new BooleanValidator("planner.enable_hashjoin_swap"); + public static final OptionValidator HASH_JOIN_SWAP_MARGIN_FACTOR = new RangeDoubleValidator("planner.join.hash_join_swap_margin_factor", 0, 100); public static final String ENABLE_DECIMAL_DATA_TYPE_KEY = "planner.enable_decimal_data_type"; - public static final BooleanValidator ENABLE_DECIMAL_DATA_TYPE = new BooleanValidator(ENABLE_DECIMAL_DATA_TYPE_KEY, false); - public static final OptionValidator HEP_OPT = new BooleanValidator("planner.enable_hep_opt", true); - public static final OptionValidator HEP_PARTITION_PRUNING = new BooleanValidator("planner.enable_hep_partition_pruning", true); + public static final BooleanValidator ENABLE_DECIMAL_DATA_TYPE = new BooleanValidator(ENABLE_DECIMAL_DATA_TYPE_KEY); + public static final OptionValidator HEP_OPT = new BooleanValidator("planner.enable_hep_opt"); + public static final OptionValidator HEP_PARTITION_PRUNING = new BooleanValidator("planner.enable_hep_partition_pruning"); public static final OptionValidator PLANNER_MEMORY_LIMIT = new RangeLongValidator("planner.memory_limit", - INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES, MAX_OFF_HEAP_ALLOCATION_IN_BYTES, DEFAULT_MAX_OFF_HEAP_ALLOCATION_IN_BYTES); + INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES, MAX_OFF_HEAP_ALLOCATION_IN_BYTES); public static final String UNIONALL_DISTRIBUTE_KEY = "planner.enable_unionall_distribute"; - public static final BooleanValidator UNIONALL_DISTRIBUTE = new BooleanValidator(UNIONALL_DISTRIBUTE_KEY, false); + public static final BooleanValidator UNIONALL_DISTRIBUTE = new BooleanValidator(UNIONALL_DISTRIBUTE_KEY); public static final OptionValidator IDENTIFIER_MAX_LENGTH = new RangeLongValidator("planner.identifier_max_length", 128 /* A minimum length is needed because option names are identifiers themselves */, - Integer.MAX_VALUE, DEFAULT_IDENTIFIER_MAX_LENGTH); + Integer.MAX_VALUE); public static final DoubleValidator FILTER_MIN_SELECTIVITY_ESTIMATE_FACTOR = new MinRangeDoubleValidator("planner.filter.min_selectivity_estimate_factor", @@ -96,15 +96,15 @@ public class PlannerSettings implements Context{ 0.0, 1.0, 1.0d, "planner.filter.min_selectivity_estimate_factor"); public static final String TYPE_INFERENCE_KEY = "planner.enable_type_inference"; - public static final BooleanValidator TYPE_INFERENCE = new BooleanValidator(TYPE_INFERENCE_KEY, true); + public static final BooleanValidator TYPE_INFERENCE = new BooleanValidator(TYPE_INFERENCE_KEY); public static final LongValidator IN_SUBQUERY_THRESHOLD = - new PositiveLongValidator("planner.in_subquery_threshold", Integer.MAX_VALUE, 20); /* Same as Calcite's default IN List subquery size */ + new PositiveLongValidator("planner.in_subquery_threshold", Integer.MAX_VALUE); /* Same as Calcite's default IN List subquery size */ - public static final String PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY = "planner.store.parquet.rowgroup.filter.pushdown"; - public static final BooleanValidator PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING = new BooleanValidator(PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY, true); + public static final String PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY = "planner.store.parquet.rowgroup.filter.pushdown.enabled"; + public static final BooleanValidator PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING = new BooleanValidator(PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_KEY); public static final String PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY = "planner.store.parquet.rowgroup.filter.pushdown.threshold"; public static final PositiveLongValidator PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD = new PositiveLongValidator(PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD_KEY, - Long.MAX_VALUE, 10000); + Long.MAX_VALUE); public static final String QUOTING_IDENTIFIERS_KEY = "planner.parser.quoting_identifiers"; public static final EnumeratedStringValidator QUOTING_IDENTIFIERS = new EnumeratedStringValidator( @@ -132,10 +132,10 @@ public class PlannerSettings implements Context{ Note: once hash and merge joins will allow non-equi join conditions, the need to turn off join optimization may go away. */ - public static final BooleanValidator JOIN_OPTIMIZATION = new BooleanValidator("planner.enable_join_optimization", true); + public static final BooleanValidator JOIN_OPTIMIZATION = new BooleanValidator("planner.enable_join_optimization"); // for testing purpose public static final String FORCE_2PHASE_AGGR_KEY = "planner.force_2phase_aggr"; - public static final BooleanValidator FORCE_2PHASE_AGGR = new BooleanValidator(FORCE_2PHASE_AGGR_KEY, false); + public static final BooleanValidator FORCE_2PHASE_AGGR = new BooleanValidator(FORCE_2PHASE_AGGR_KEY); public OptionManager options = null; public FunctionImplementationRegistry functionImplementationRegistry = null; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java index e03a40c60bc..bdf12dde5b4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java @@ -448,8 +448,8 @@ protected Prel convertToPrel(RelNode drel, RelDataType validatedRowType) throws log("Not enough memory for this plan", phyRelNode, logger, null); logger.debug("Re-planning without hash operations."); - queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHJOIN.getOptionName(), false)); - queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHAGG.getOptionName(), false)); + queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHJOIN.getOptionName(), false, OptionValue.OptionScope.QUERY)); + queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHAGG.getOptionName(), false, OptionValue.OptionScope.QUERY)); try { final RelNode relNode = transform(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, drel, traits); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java index f278989943d..95115f65974 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SetOptionHandler.java @@ -31,6 +31,7 @@ import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import org.apache.drill.exec.util.ImpersonationUtil; import org.apache.drill.exec.work.foreman.ForemanSetupException; import org.apache.calcite.sql.SqlLiteral; @@ -64,15 +65,19 @@ public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, Foreman final String scope = option.getScope(); final OptionValue.OptionType type; + final OptionValue.OptionScope optionScope; if (scope == null) { // No scope mentioned assumed SESSION type = OptionType.SESSION; + optionScope = OptionScope.SESSION; } else { switch (scope.toLowerCase()) { case "session": type = OptionType.SESSION; + optionScope = OptionScope.SESSION; break; case "system": type = OptionType.SYSTEM; + optionScope = OptionScope.SYSTEM; break; default: throw UserException.validationError() @@ -99,7 +104,7 @@ public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, Foreman // Currently, we convert multi-part identifier to a string. final String name = option.getName().toString(); if (value != null) { // SET option - final OptionValue optionValue = createOptionValue(name, type, (SqlLiteral) value); + final OptionValue optionValue = createOptionValue(name, type, (SqlLiteral) value, optionScope); options.setOption(optionValue); } else { // RESET option if ("ALL".equalsIgnoreCase(name)) { @@ -113,36 +118,36 @@ public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, Foreman } private static OptionValue createOptionValue(final String name, final OptionValue.OptionType type, - final SqlLiteral literal) { + final SqlLiteral literal, final OptionValue.OptionScope scope) { final Object object = literal.getValue(); final SqlTypeName typeName = literal.getTypeName(); switch (typeName) { case DECIMAL: { final BigDecimal bigDecimal = (BigDecimal) object; if (bigDecimal.scale() == 0) { - return OptionValue.createLong(type, name, bigDecimal.longValue()); + return OptionValue.createLong(type, name, bigDecimal.longValue(), scope); } else { - return OptionValue.createDouble(type, name, bigDecimal.doubleValue()); + return OptionValue.createDouble(type, name, bigDecimal.doubleValue(), scope); } } case DOUBLE: case FLOAT: - return OptionValue.createDouble(type, name, ((BigDecimal) object).doubleValue()); + return OptionValue.createDouble(type, name, ((BigDecimal) object).doubleValue(), scope); case SMALLINT: case TINYINT: case BIGINT: case INTEGER: - return OptionValue.createLong(type, name, ((BigDecimal) object).longValue()); + return OptionValue.createLong(type, name, ((BigDecimal) object).longValue(), scope); case VARBINARY: case VARCHAR: case CHAR: - return OptionValue.createString(type, name, ((NlsString) object).getValue()); + return OptionValue.createString(type, name, ((NlsString) object).getValue(), scope); case BOOLEAN: - return OptionValue.createBoolean(type, name, (Boolean) object); + return OptionValue.createBoolean(type, name, (Boolean) object, scope); default: throw UserException.validationError() diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java index b64ed14c629..9c94065f6fa 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/InboundImpersonationManager.java @@ -86,8 +86,8 @@ private static List deserializeImpersonationPolicies(final */ public static class InboundImpersonationPolicyValidator extends StringValidator { - public InboundImpersonationPolicyValidator(String name, String def) { - super(name, def, true); + public InboundImpersonationPolicyValidator(String name) { + super(name, true); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java index fc647c8c1da..c30d1b35895 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java @@ -41,6 +41,8 @@ import org.apache.drill.exec.proto.UserProtos.UserProperties; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; +import org.apache.drill.exec.server.options.OptionValue.OptionType; import org.apache.drill.exec.server.options.SessionOptionManager; import com.google.common.collect.Maps; @@ -247,8 +249,8 @@ public SchemaPlus getDefaultSchema(SchemaPlus rootSchema) { * @param value option value */ public void setSessionOption(String name, String value) { - OptionValue.Kind optionKind = SystemOptionManager.getValidator(name).getKind(); - OptionValue optionValue = OptionValue.createOption(optionKind, OptionValue.OptionType.SESSION, name, value); + OptionValue.Kind optionKind = ((SessionOptionManager) sessionOptions).getFallbackOptionManager().getValidator(name).getKind(); + OptionValue optionValue = OptionValue.createOption(optionKind, OptionType.SESSION, name, value, OptionScope.SESSION); sessionOptions.setOption(optionValue); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java index 7ed92208dea..6f22b76f9e0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java @@ -34,6 +34,7 @@ import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import org.apache.drill.exec.server.rest.WebServer; import org.apache.drill.exec.service.ServiceEngine; import org.apache.drill.exec.store.StoragePluginRegistry; @@ -226,7 +227,7 @@ private void javaPropertiesToSystemOptions() { } final OptionValue optionValue = OptionValue.createOption( - defaultValue.kind, OptionType.SYSTEM, optionName, optionString); + defaultValue.kind, OptionType.SYSTEM, optionName, optionString, OptionScope.SYSTEM); optionManager.setOption(optionValue); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java index 973b97ca206..fb3ccf8f09a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java @@ -104,7 +104,7 @@ public DrillbitContext( this.reader = new PhysicalPlanReader(context.getConfig(), classpathScan, lpPersistence, endpoint, storagePlugins); this.operatorCreatorRegistry = new OperatorCreatorRegistry(classpathScan); - this.systemOptions = new SystemOptionManager(lpPersistence, provider); + this.systemOptions = new SystemOptionManager(lpPersistence, provider,context.getConfig()); this.functionRegistry = new FunctionImplementationRegistry(context.getConfig(), classpathScan, systemOptions); this.compiler = new CodeCompiler(context.getConfig(), systemOptions); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java index 299b22148b7..9eba2a34ff6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/BaseOptionManager.java @@ -56,4 +56,5 @@ public long getOption(LongValidator validator) { public String getOption(StringValidator validator) { return getOptionSafe(validator).string_val; } + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/DrillConfigIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/DrillConfigIterator.java index 34ebf80f10e..2b3fdfc9952 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/DrillConfigIterator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/DrillConfigIterator.java @@ -22,6 +22,7 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import com.typesafe.config.ConfigValue; @@ -57,17 +58,17 @@ public OptionValue next() { OptionValue optionValue = null; switch(cv.valueType()) { case BOOLEAN: - optionValue = OptionValue.createBoolean(OptionType.BOOT, name, (Boolean) cv.unwrapped()); + optionValue = OptionValue.createBoolean(OptionType.BOOT, name, (Boolean) cv.unwrapped(), OptionScope.BOOT); break; case LIST: case OBJECT: case STRING: - optionValue = OptionValue.createString(OptionType.BOOT, name, cv.render()); + optionValue = OptionValue.createString(OptionType.BOOT, name, cv.render(),OptionScope.BOOT); break; case NUMBER: - optionValue = OptionValue.createLong(OptionType.BOOT, name, ((Number) cv.unwrapped()).longValue()); + optionValue = OptionValue.createLong(OptionType.BOOT, name, ((Number) cv.unwrapped()).longValue(),OptionScope.BOOT); break; case NULL: diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java index 373b0d2a4e3..301fa34d898 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/FallbackOptionManager.java @@ -108,7 +108,7 @@ public OptionValue getOption(final String name) { @Override public void setOption(OptionValue value) { - final OptionValidator validator = SystemOptionManager.getValidator(value.name); + final OptionValidator validator = getSystemOptionManager().getValidator(value.name); validator.validate(value, this); // validate the option @@ -120,7 +120,7 @@ public void setOption(OptionValue value) { @Override public void deleteOption(final String name, final OptionType type) { - SystemOptionManager.getValidator(name); // ensure the option exists + getSystemOptionManager().getValidator(name); // ensure the option exists // fallback if unable to delete locally if (!deleteLocalOption(name, type)) { @@ -144,4 +144,27 @@ public OptionList getOptionList() { } return list; } + + public OptionManager getFallback() { + return fallback; + } + + /** + * {@link FragmentOptionManager} and {@link SessionOptionManager} use {@link SystemOptionManager} as the fall back + * manager so for both FragmentOptionManager and SessionOptionManager fallback is the SystemOptionManager so it is + * returned. But in case of {@link QueryOptionManager}, it uses {@link SessionOptionManager} as the fallback manager + * and since SessionOptionManager uses SystemOptionManager as fallback, SystemOptionManager can be fetched from the + * SessionOptionManager. + */ + public SystemOptionManager getSystemOptionManager() { + final SystemOptionManager systemOptionManager; + if(fallback instanceof SessionOptionManager) { + final SessionOptionManager sessionOptionManager = (SessionOptionManager) fallback; + systemOptionManager = sessionOptionManager.getFallbackOptionManager(); + } + else { + systemOptionManager = (SystemOptionManager) fallback; + } + return systemOptionManager; + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java index 951cbc4e97a..7122a19956e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValidator.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.server.options; +import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.server.options.OptionValue.Kind; @@ -28,7 +29,7 @@ public abstract class OptionValidator { // the error messages produced by the validator private final String optionName; private final boolean isAdminOption; - + public static final String OPTION_DEFAULTS_ROOT = "drill.exec.options."; /** By default, if admin option value is not specified, it would be set to false.*/ public OptionValidator(String optionName) { this(optionName, false); @@ -106,4 +107,11 @@ public boolean isAdminOption() { */ public abstract Kind getKind(); + /** + * Loads the default option value for this validator. + * + * @return default option value + */ + public abstract void loadDefault(DrillConfig bootConfig); + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java index 3c07608c96e..588348027e3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java @@ -22,14 +22,22 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; import org.apache.drill.exec.store.sys.PersistentStore; +import com.google.common.base.Preconditions; /** + *

* An {@link OptionValue option value} is used by an {@link OptionManager} to store a run-time setting. This setting, * for example, could affect a query in execution stage. Instances of this class are JSON serializable and can be stored * in a {@link PersistentStore persistent store} (see {@link SystemOptionManager#options}), or * in memory (see {@link InMemoryOptionManager#options}). + *

+ *

OptionType defines the permission level of the option.If it can be set at System level or Session level or so on. + * Whereas OptionScope defines the scope at which the option is being set. If the option is being set at the BOOT time + * the scope of the option is BOOT. If it is set at SYSTEM level the scope is SYSTEM. Although they look similar there + * is a fine level which differentiates both of them which is at which level of hierarchy they can be set and + * at what at level of hierarchy they were actually set. + *

*/ @JsonInclude(Include.NON_NULL) public class OptionValue implements Comparable { @@ -42,6 +50,10 @@ public enum Kind { BOOLEAN, LONG, STRING, DOUBLE } + public enum OptionScope { + BOOT, SYSTEM, SESSION, QUERY + } + public final String name; public final Kind kind; public final OptionType type; @@ -49,33 +61,34 @@ public enum Kind { public final String string_val; public final Boolean bool_val; public final Double float_val; + public final OptionScope scope; - public static OptionValue createLong(OptionType type, String name, long val) { - return new OptionValue(Kind.LONG, type, name, val, null, null, null); + public static OptionValue createLong(OptionType type, String name, long val, OptionScope scope) { + return new OptionValue(Kind.LONG, type, name, val, null, null, null, scope); } - public static OptionValue createBoolean(OptionType type, String name, boolean bool) { - return new OptionValue(Kind.BOOLEAN, type, name, null, null, bool, null); + public static OptionValue createBoolean(OptionType type, String name, boolean bool, OptionScope scope) { + return new OptionValue(Kind.BOOLEAN, type, name, null, null, bool, null, scope); } - public static OptionValue createString(OptionType type, String name, String val) { - return new OptionValue(Kind.STRING, type, name, null, val, null, null); + public static OptionValue createString(OptionType type, String name, String val, OptionScope scope) { + return new OptionValue(Kind.STRING, type, name, null, val, null, null, scope); } - public static OptionValue createDouble(OptionType type, String name, double val) { - return new OptionValue(Kind.DOUBLE, type, name, null, null, null, val); + public static OptionValue createDouble(OptionType type, String name, double val, OptionScope scope) { + return new OptionValue(Kind.DOUBLE, type, name, null, null, null, val, scope); } - public static OptionValue createOption(Kind kind, OptionType type, String name, String val) { + public static OptionValue createOption(Kind kind, OptionType type, String name, String val, OptionScope scope) { switch (kind) { case BOOLEAN: - return createBoolean(type, name, Boolean.valueOf(val)); + return createBoolean(type, name, Boolean.valueOf(val), scope); case LONG: - return createLong(type, name, Long.valueOf(val)); + return createLong(type, name, Long.valueOf(val), scope); case STRING: - return createString(type, name, val); + return createString(type, name, val, scope); case DOUBLE: - return createDouble(type, name, Double.valueOf(val)); + return createDouble(type, name, Double.valueOf(val), scope); default: return null; } @@ -88,16 +101,17 @@ private OptionValue(@JsonProperty("kind") Kind kind, @JsonProperty("num_val") Long num_val, @JsonProperty("string_val") String string_val, @JsonProperty("bool_val") Boolean bool_val, - @JsonProperty("float_val") Double float_val) { + @JsonProperty("float_val") Double float_val, + @JsonProperty("scope") OptionScope scope) { Preconditions.checkArgument(num_val != null || string_val != null || bool_val != null || float_val != null); this.kind = kind; this.type = type; this.name = name; - this.float_val = float_val; this.num_val = num_val; this.string_val = string_val; this.bool_val = bool_val; + this.scope = scope; } public String getName() { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java index 38f85568576..512c8e925cc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java @@ -61,7 +61,7 @@ boolean setLocalOption(final OptionValue value) { return false; } final String name = value.name; - final OptionValidator validator = SystemOptionManager.getValidator(name); // if set, validator must exist. + final OptionValidator validator = getFallbackOptionManager().getValidator(name); // if set, validator must exist. final boolean shortLived = validator.isShortLived(); if (shortLived) { final int start = session.getQueryCount() + 1; // start from the next query @@ -83,7 +83,7 @@ OptionValue getLocalOption(final String name) { final int start = shortLivedOptions.get(name).getLeft(); // option is not in effect if queryNumber < start if (queryNumber < start) { - return SystemOptionManager.getValidator(name).getDefault(); + return getFallbackOptionManager().getValidator(name).getDefault(); // reset if queryNumber <= end } else { options.remove(name); @@ -120,4 +120,10 @@ Iterable getLocalOptions() { boolean supportsOptionType(OptionType type) { return type == OptionType.SESSION; } + + /* Gets fallback manager and returns it as SystemOptionManager */ + public SystemOptionManager getFallbackOptionManager() { + final SystemOptionManager systemOptionManager = (SystemOptionManager) getFallback(); + return systemOptionManager; + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index 3392a216099..d2dfc2a5dac 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -26,7 +26,9 @@ import java.util.Map.Entry; import java.util.Set; +import com.typesafe.config.ConfigException; import org.apache.commons.collections.IteratorUtils; +import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.map.CaseInsensitiveMap; @@ -48,12 +50,31 @@ * Only one instance of this class exists per drillbit. Options set at the system level affect the entire system and * persist between restarts. */ + +/** + *

All the system options are externalized into conf file. While adding a new system option + * a validator should be added and the default value for the option should be set in + * the conf files(example : drill-module.conf) under the namespace drill.exec.options. + *

+ * + *

+ * The SystemOptionManager loads all the validators and the default values for the options are + * fetched from the config. The validators are populated with the default values fetched from + * the config. If the option is not set in the conf files config option is missing exception + * will be thrown. + *

+ * + *

+ * If the option is set using ALTER, the value that is set will be returned. Else the default value + * that is loaded into validator from the config will be returned. + *

+ */ + public class SystemOptionManager extends BaseOptionManager implements OptionManager, AutoCloseable { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SystemOptionManager.class); - private static final CaseInsensitiveMap VALIDATORS; - - static { + private CaseInsensitiveMap VALIDATORS; + public void populateValidators() { final OptionValidator[] validators = new OptionValidator[]{ PlannerSettings.CONSTANT_FOLDING, PlannerSettings.EXCHANGE, @@ -177,8 +198,9 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana ExecConstants.USE_DYNAMIC_UDFS, ExecConstants.QUERY_TRANSIENT_STATE_UPDATE, ExecConstants.PERSISTENT_TABLE_UMASK_VALIDATOR, - ExecConstants.ENABLE_ITERATOR_VALIDATOR, - ExecConstants.ENABLE_VECTOR_VALIDATOR + ExecConstants.CPU_LOAD_AVERAGE, + ExecConstants.ENABLE_VECTOR_VALIDATOR, + ExecConstants.ENABLE_ITERATOR_VALIDATOR }; final Map tmp = new HashMap<>(); for (final OptionValidator validator : validators) { @@ -194,6 +216,7 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana private final PersistentStoreProvider provider; + private final DrillConfig bootConfig; /** * Persistent store for options that have been changed from default. * NOTE: CRUD operations must use lowercase keys. @@ -201,10 +224,35 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana private PersistentStore options; public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider) { + this(lpPersistence,provider,null); + } + + public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider, final DrillConfig bootConfig) { this.provider = provider; - this.config = PersistentStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), OptionValue.class) - .name("sys.options") - .build(); + this.config = PersistentStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), OptionValue.class) + .name("sys.options") + .build(); + this.bootConfig = bootConfig; + populateValidators(); + populateDefaultValues(); + + } + + /** + * Gets the {@link OptionValidator} associated with the name. + * + * @param name name of the option + * @return the associated validator + * @throws UserException - if the validator is not found + */ + public OptionValidator getValidator(final String name) { + final OptionValidator validator = VALIDATORS.get(name); + if (validator == null) { + throw UserException.validationError() + .message(String.format("The option '%s' does not exist.", name.toLowerCase())) + .build(logger); + } + return validator; } /** @@ -237,23 +285,6 @@ public SystemOptionManager init() throws Exception { return this; } - /** - * Gets the {@link OptionValidator} associated with the name. - * - * @param name name of the option - * @return the associated validator - * @throws UserException - if the validator is not found - */ - public static OptionValidator getValidator(final String name) { - final OptionValidator validator = VALIDATORS.get(name); - if (validator == null) { - throw UserException.validationError() - .message(String.format("The option '%s' does not exist.", name)) - .build(logger); - } - return validator; - } - @Override public Iterator iterator() { final Map buildList = CaseInsensitiveMap.newHashMap(); @@ -276,7 +307,7 @@ public OptionValue getOption(final String name) { return value; } - // otherwise, return default. + // If option is not set return the default set in the validator. final OptionValidator validator = getValidator(name); return validator.getDefault(); } @@ -288,7 +319,6 @@ public void setOption(final OptionValue value) { final OptionValidator validator = getValidator(name); validator.validate(value, this); // validate the option - if (options.get(name) == null && value.equals(validator.getDefault())) { return; // if the option is not overridden, ignore setting option to default } @@ -315,6 +345,19 @@ public void deleteAllOptions(OptionType type) { } } + public void populateDefaultValues() { + + // populate the options from the config + final Map tmp = new HashMap<>(); + for (OptionValidator validator: VALIDATORS.values()) { + try { + validator.loadDefault(bootConfig); + } catch (ConfigException.Missing e) { + throw new IllegalStateException("Config Option is missing"+ validator.getOptionName()); + } + } + } + @Override public OptionList getOptionList() { return (OptionList) IteratorUtils.toList(iterator()); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java index 3604eb76f6b..ecaa7b70b30 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java @@ -20,10 +20,11 @@ import java.util.Set; import com.google.common.collect.Sets; +import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.server.options.OptionValue.Kind; import org.apache.drill.exec.server.options.OptionValue.OptionType; - +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import static com.google.common.base.Preconditions.checkArgument; public class TypeValidators { @@ -31,8 +32,8 @@ public class TypeValidators { public static class PositiveLongValidator extends LongValidator { private final long max; - public PositiveLongValidator(String name, long max, long def) { - super(name, def); + public PositiveLongValidator(String name, long max) { + super(name); this.max = max; } @@ -49,8 +50,8 @@ public void validate(final OptionValue v, final OptionSet manager) { public static class PowerOfTwoLongValidator extends PositiveLongValidator { - public PowerOfTwoLongValidator(String name, long max, long def) { - super(name, max, def); + public PowerOfTwoLongValidator(String name, long max) { + super(name, max); } @Override @@ -72,8 +73,8 @@ public static class RangeDoubleValidator extends DoubleValidator { private final double min; private final double max; - public RangeDoubleValidator(String name, double min, double max, double def) { - super(name, def); + public RangeDoubleValidator(String name, double min, double max) { + super(name); this.min = min; this.max = max; } @@ -93,7 +94,7 @@ public static class MinRangeDoubleValidator extends RangeDoubleValidator { private final String maxValidatorName; public MinRangeDoubleValidator(String name, double min, double max, double def, String maxValidatorName) { - super(name, min, max, def); + super(name, min, max); this.maxValidatorName = maxValidatorName; } @@ -114,7 +115,7 @@ public static class MaxRangeDoubleValidator extends RangeDoubleValidator { private final String minValidatorName; public MaxRangeDoubleValidator(String name, double min, double max, double def, String minValidatorName) { - super(name, min, max, def); + super(name, min, max); this.minValidatorName = minValidatorName; } @@ -132,42 +133,61 @@ public void validate(final OptionValue v, final OptionSet manager) { } public static class BooleanValidator extends TypeValidator { - public BooleanValidator(String name, boolean def) { - this(name, def, false); + public BooleanValidator(String name) { + this(name, false); + } + + public BooleanValidator(String name, boolean isAdminOption) { + super(name, Kind.BOOLEAN, isAdminOption); } - public BooleanValidator(String name, boolean def, boolean isAdminOption) { - super(name, Kind.BOOLEAN, OptionValue.createBoolean(OptionType.SYSTEM, name, def), isAdminOption); + public void loadDefault(DrillConfig bootConfig){ + OptionValue value = OptionValue.createBoolean(OptionType.SYSTEM, getOptionName(), bootConfig.getBoolean(getConfigProperty()), OptionScope.BOOT); + setDefaultValue(value); } } public static class StringValidator extends TypeValidator { - public StringValidator(String name, String def) { - this(name, def, false); + public StringValidator(String name) { + this(name, false); + } + public StringValidator(String name, boolean isAdminOption) { + super(name, Kind.STRING, isAdminOption); } - public StringValidator(String name, String def, boolean isAdminOption) { - super(name, Kind.STRING, OptionValue.createString(OptionType.SYSTEM, name, def), isAdminOption); + public void loadDefault(DrillConfig bootConfig){ + OptionValue value = OptionValue.createString(OptionType.SYSTEM, getOptionName(), bootConfig.getString(getConfigProperty()), OptionScope.BOOT); + setDefaultValue(value); } } public static class LongValidator extends TypeValidator { - public LongValidator(String name, long def) { - this(name, def, false); + public LongValidator(String name) { + this(name, false); } - public LongValidator(String name, long def, boolean isAdminOption) { - super(name, Kind.LONG, OptionValue.createLong(OptionType.SYSTEM, name, def), isAdminOption); + public LongValidator(String name, boolean isAdminOption) { + super(name, Kind.LONG, isAdminOption); + } + + public void loadDefault(DrillConfig bootConfig){ + OptionValue value = OptionValue.createLong(OptionType.SYSTEM, getOptionName(), bootConfig.getLong(getConfigProperty()), OptionScope.BOOT); + setDefaultValue(value); } } public static class DoubleValidator extends TypeValidator { - public DoubleValidator(String name, double def) { - this(name, def, false); + public DoubleValidator(String name) { + this(name,false); } - public DoubleValidator(String name, double def, boolean isAdminOption) { - super(name, Kind.DOUBLE, OptionValue.createDouble(OptionType.SYSTEM, name, def), isAdminOption); + public DoubleValidator(String name, boolean isAdminOption) { + super(name, Kind.DOUBLE, isAdminOption); + } + + public void loadDefault(DrillConfig bootConfig){ + OptionValue value = OptionValue.createDouble(OptionType.SYSTEM, getOptionName(),bootConfig.getDouble(getConfigProperty()), OptionScope.BOOT); + setDefaultValue(value); } } @@ -175,8 +195,8 @@ public static class RangeLongValidator extends LongValidator { private final long min; private final long max; - public RangeLongValidator(String name, long min, long max, long def) { - super(name, def); + public RangeLongValidator(String name, long min, long max) { + super(name); this.min = min; this.max = max; } @@ -198,9 +218,8 @@ public void validate(final OptionValue v, final OptionSet manager) { public static class EnumeratedStringValidator extends StringValidator { private final Set valuesSet = Sets.newLinkedHashSet(); - public EnumeratedStringValidator(String name, String def, String... values) { - super(name, def); - valuesSet.add(def.toLowerCase()); + public EnumeratedStringValidator(String name, String... values) { + super(name); for (String value : values) { valuesSet.add(value.toLowerCase()); } @@ -217,19 +236,51 @@ public void validate(final OptionValue v, final OptionSet manager) { } } + /** Max width is a special validator which computes and validates + * the maxwidth. If the maxwidth is already set in system/session + * the value is returned or else it is computed dynamically based on + * the available number of processors and cpu load average + */ + public static class MaxWidthValidator extends LongValidator{ + + public MaxWidthValidator(String name) { + this(name, false); + } + + public MaxWidthValidator(String name, boolean isAdminOption) { + super(name, isAdminOption); + } + + public void loadDefault(DrillConfig bootConfig) { + OptionValue value = OptionValue.createLong(OptionType.SYSTEM, getOptionName(), bootConfig.getLong(getConfigProperty()), OptionScope.BOOT); + setDefaultValue(value); + } + + public int computeMaxWidth(double cpuLoadAverage, long maxWidth) { + // if maxwidth is already set return it + if (maxWidth != 0) { + return (int) maxWidth; + } + // else compute the value and return + else { + int availProc = Runtime.getRuntime().availableProcessors(); + long maxWidthPerNode = Math.max(1, Math.min(availProc, Math.round(availProc * cpuLoadAverage))); + return (int) maxWidthPerNode; + } + } + } + public static abstract class TypeValidator extends OptionValidator { private final Kind kind; - private final OptionValue defaultValue; + private OptionValue defaultValue = null; - public TypeValidator(final String name, final Kind kind, final OptionValue defValue) { - this(name, kind, defValue, false); + public TypeValidator(final String name, final Kind kind) { + this(name, kind, false); } - public TypeValidator(final String name, final Kind kind, final OptionValue defValue, final boolean isAdminOption) { + public TypeValidator(final String name, final Kind kind, final boolean isAdminOption) { super(name, isAdminOption); - checkArgument(defValue.type == OptionType.SYSTEM, "Default value must be SYSTEM type."); this.kind = kind; - this.defaultValue = defValue; } @Override @@ -256,5 +307,13 @@ public void validate(final OptionValue v, final OptionSet manager) { public Kind getKind() { return kind; } + + protected void setDefaultValue(OptionValue defaultValue) { + this.defaultValue = defaultValue; + } + + public String getConfigProperty() { + return OPTION_DEFAULTS_ROOT + getOptionName(); + } } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java index d0007d366fd..1caf10b7442 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java @@ -102,7 +102,7 @@ public Viewable updateSystemOption(@FormParam("name") String name, @FormParam("v OptionValue.Kind.valueOf(kind), OptionValue.OptionType.SYSTEM, name, - value)); + value, OptionValue.OptionScope.SYSTEM)); } catch (Exception e) { logger.debug("Could not update.", e); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java new file mode 100644 index 00000000000..fdd5092c8dd --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.sys; + +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + + +import com.google.common.collect.Lists; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.server.options.OptionManager; +import org.apache.drill.exec.server.options.OptionValue; +import org.apache.drill.exec.server.options.OptionValue.Kind; +import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; + + +/* + * Extends the original Option iterator. The idea is to hide the implementation details and present the + * user with the rows which have values set at the top level of hierarchy and exclude the values set + * at lower levels. This is done by examining the scope and the precedence order of scope is session - system - default. + * All the values are represented as String instead of having multiple + * columns and the data type is provided as kind to preserve type information about the option. + * The query output is as follows - + * name,kind,type,val,optionScope + * planner.slice_target,BIGINT,SESSION,20,SESSION + * planner.width.max_per_node,BIGINT,SYSTEM,0,BOOT + * planner.affinity_factor,FLOAT,SYSTEM,1.7,SYSTEM + * In the above example, the query output contains single row for each option + * and we can infer that slice target is set at the session level and max width + * per node is set at the BOOT level and affinity factor is set at the SYSTEM level. + * The options set in the top level of hierarchy always takes precedence and they are returned + * in the query output. For example if the option is set at both SESSION level and + * SYSTEM level the value set at SESSION level takes precedence and query output has + * only the value set at SESSION level. + */ +public class ExtendedOptionIterator implements Iterator { +// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OptionIterator.class); + + private final OptionManager fragmentOptions; + private final Iterator mergedOptions; + + public ExtendedOptionIterator(FragmentContext context) { + fragmentOptions = context.getOptions(); + mergedOptions = sortOptions(fragmentOptions.iterator()); + } + /* * + * Remove the redundant rows for the same option based on the scope and return + * the value that takes precedence over others. For example the option set in session + * scope takes precedence over system and boot and etc., + */ + public Iterator sortOptions(Iterator options) { + List optionslist = Lists.newArrayList(options); + HashMap optionsmap = new HashMap<>(); + final Map preference = new HashMap() {{ + put(OptionScope.SESSION, 0); + put(OptionScope.SYSTEM, 1); + put(OptionScope.BOOT, 2); + }}; + + for (OptionValue option : optionslist) { + if (optionsmap.containsKey(option.getName())) { + + if (preference.get(option.scope) < preference.get(optionsmap.get(option.getName()).scope)) { + optionsmap.put(option.getName(), option); + } + + } else { + optionsmap.put(option.getName(), option); + } + } + optionslist.clear(); + for (String name : optionsmap.keySet()) { + optionslist.add(optionsmap.get(name)); + } + + Collections.sort(optionslist, new Comparator() { + @Override + public int compare(OptionValue v1, OptionValue v2) { + return v1.name.compareTo(v2.name); + } + }); + + return optionslist.iterator(); + } + + @Override + public boolean hasNext() { + return mergedOptions.hasNext(); + } + + @Override + public ExtendedOptionValueWrapper next() { + final OptionValue value = mergedOptions.next(); + final HashMap typeMapping = new HashMap() {{ + put(Kind.STRING,"VARCHAR"); + put(Kind.DOUBLE,"FLOAT"); + put(Kind.LONG,"BIGINT"); + put(Kind.BOOLEAN,"BIT"); + + }}; + return new ExtendedOptionValueWrapper(value.name, typeMapping.get(value.kind), value.type,value.getValue().toString(), value.scope); + } + + public enum Status { + BOOT, DEFAULT, CHANGED + } + + /** + * Wrapper class for Extended Option Value + */ + public static class ExtendedOptionValueWrapper { + + public final String name; + public final String kind; + public final OptionType type; + public final String val; + public final OptionScope optionScope; + + + public ExtendedOptionValueWrapper(final String name, final String kind, final OptionType type, final String value, final OptionScope scope) { + this.name = name; + this.kind = kind; + this.type = type; + this.val = value; + this.optionScope = scope; + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } +} + + diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java index 5c8a6412dcd..424ff959e3b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java @@ -25,6 +25,7 @@ import com.google.common.collect.Lists; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.server.options.DrillConfigIterator; +import org.apache.drill.exec.server.options.FragmentOptionManager; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.OptionValue.Kind; @@ -40,10 +41,12 @@ enum Mode { private final OptionManager fragmentOptions; private final Iterator mergedOptions; + private FragmentContext context; public OptionIterator(FragmentContext context, Mode mode){ final DrillConfigIterator configOptions = new DrillConfigIterator(context.getConfig()); fragmentOptions = context.getOptions(); + this.context = context; final Iterator optionList; switch(mode){ case BOOT: @@ -71,10 +74,12 @@ public boolean hasNext() { public OptionValueWrapper next() { final OptionValue value = mergedOptions.next(); final Status status; + final FragmentOptionManager fragmentOptionManager = (FragmentOptionManager) fragmentOptions; + final SystemOptionManager systemOptionManager = (SystemOptionManager) fragmentOptionManager.getFallback(); if (value.type == OptionType.BOOT) { status = Status.BOOT; } else { - final OptionValue def = SystemOptionManager.getValidator(value.name).getDefault(); + final OptionValue def = systemOptionManager.getValidator(value.name).getDefault(); if (value.equalsIgnoreType(def)) { status = Status.DEFAULT; } else { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java index 7f99fb764a4..fa6981b5fe3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java @@ -21,6 +21,7 @@ import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.store.sys.OptionIterator.OptionValueWrapper; +import org.apache.drill.exec.store.sys.ExtendedOptionIterator.ExtendedOptionValueWrapper; /** * An enumeration of all tables in Drill's system ("sys") schema. @@ -39,6 +40,13 @@ public Iterator getIterator(final FragmentContext context) { } }, + OPTION2("options2", false,ExtendedOptionIterator.ExtendedOptionValueWrapper.class ) { + @Override + public Iterator getIterator(final FragmentContext context) { + return new ExtendedOptionIterator(context); + } + }, + BOOT("boot", false, OptionValueWrapper.class) { @Override public Iterator getIterator(final FragmentContext context) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java index 000d90f7158..a046c774bbf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.ExecConstants; @@ -75,13 +76,11 @@ public static class ControlsOptionValidator extends TypeValidator { /** * Constructor for controls option validator. - * - * @param name the name of the validator - * @param def the default JSON, specified as string + * @param name the name of the validator * @param ttl the number of queries for which this option should be valid */ - public ControlsOptionValidator(final String name, final String def, final int ttl) { - super(name, OptionValue.Kind.STRING, OptionValue.createString(OptionType.SYSTEM, name, def)); + public ControlsOptionValidator(final String name, final int ttl) { + super(name, OptionValue.Kind.STRING); assert ttl > 0; this.ttl = ttl; } @@ -112,6 +111,11 @@ public void validate(final OptionValue v, final OptionSet manager) { .build(logger); } } + + public void loadDefault(DrillConfig bootConfig){ + OptionValue value = OptionValue.createString(OptionType.SYSTEM, getOptionName(), bootConfig.getString(getConfigProperty()), OptionValue.OptionScope.BOOT); + setDefaultValue(value); + } } /** diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java index 45802227ab9..48724a445d0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java @@ -58,7 +58,9 @@ public static void setupBufferedOpsMemoryAllocations(final PhysicalPlan plan, fi // if there are any sorts, compute the maximum allocation, and set it on them if (bufferedOpList.size() > 0) { final OptionManager optionManager = queryContext.getOptions(); - final long maxWidthPerNode = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val; + double cpu_load_average = optionManager.getOption(ExecConstants.CPU_LOAD_AVERAGE); + final long maxWidth = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE); + final long maxWidthPerNode = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(cpu_load_average,maxWidth); long maxAllocPerNode = Math.min(DrillConfig.getMaxDirectMemory(), queryContext.getConfig().getLong(RootAllocatorFactory.TOP_LEVEL_MAX_ALLOC)); maxAllocPerNode = Math.min(maxAllocPerNode, diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index 41ecc95a421..f387cda40f9 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -349,3 +349,150 @@ drill.exec: { drill.jdbc: { batch_queue_throttling_threshold: 100 } + +# The following are defaults for system and session options. +# Provided here for easy customization in Drill distributions +# like the drill-distrib.conf file. +# An addition of new system option should go through following procedure. +# Before adding a system option a validator should be added for that +# option and then the option should be configured below under the name +# space drill.exec.options. When the system options are being loaded +# options config options are read to find the default value for a give option +# if the option is not set using ALTER SYSTEM or ALTER SESSION.But +# if the option is not configured in the conf file under the name space +# "drill.exec.options", option missing exception will be thrown. +# Users are not supposed to set these options in the drill-override.conf file. +# Users should use ALTER SYSTEM and ALTER SESSION to set the options. + +drill.exec.options: { + bootstrap-storage-plugins.json: .sys.drill, + debug.validate_iterators : false, + debug.validate_vectors :false, + drill.exec.functions.cast_empty_string_to_null: false, + drill.exec.hashagg.min_batches_per_partition : 3, + drill.exec.storage.file.partition.column.label: "dir", + drill.exec.storage.implicit.filename.column.label: "filename", + drill.exec.storage.implicit.filepath.column.label: "filepath", + drill.exec.storage.implicit.fqn.column.label: "fqn", + drill.exec.storage.implicit.suffix.column.label: "suffix", + drill.exec.testing.controls: "{}", + exec.bulk_load_table_list.bulk_size: 1000, + exec.compile.scalar_replacement: false, + exec.enable_bulk_load_table_list: false, + exec.enable_union_type: false, + exec.errors.verbose: false, + exec.hashagg.mem_limit : 0, + exec.hashagg.num_partitions :32, + exec.impersonation.inbound_policies: "[]", + exec.java.compiler.exp_in_method_size: 50, + exec.java_compiler : "DEFAULT", + exec.java_compiler_debug :true, + exec.java_compiler_janino_maxsize : 262144, + exec.max_hash_table_size: 1073741824, + exec.min_hash_table_size: 65536, + exec.persistent_table.umask: "002", + exec.query.progress.update: true, + exec.query_profile.debug_mode: false, + exec.query_profile.save: true, + exec.queue.enable: false, + exec.queue.large: 10, + exec.queue.small: 100, + exec.queue.threshold: 30000000, + exec.queue.timeout_millis: 300000, + exec.sort.disable_managed : false, + exec.storage.enable_new_text_reader: true, + exec.udf.enable_dynamic_support: true, + exec.udf.use_dynamic: true, + new_view_default_permissions: 700, + org.apache.drill.exec.compile.ClassTransformer.scalar_replacement : "try", + planner.add_producer_consumer:false, + planner.affinity_factor: 1.2, + planner.broadcast_factor:1.0, + planner.broadcast_threshold:10000000, + planner.cpu_load_average : 0.70, + planner.disable_exchanges:false, + planner.enable_broadcast_join:true, + planner.enable_constant_folding:true, + planner.enable_decimal_data_type:false, + planner.enable_demux_exchange:false, + planner.enable_hash_single_key:true, + planner.enable_hashagg:true, + planner.enable_hashjoin:true, + planner.enable_hashjoin_swap:true, + planner.enable_hep_opt:true, + planner.enable_hep_partition_pruning:true, + planner.enable_join_optimization: true, + planner.enable_limit0_optimization: false, + planner.enable_mergejoin:true, + planner.enable_multiphase_agg:true, + planner.enable_mux_exchange:true, + planner.enable_nestedloopjoin:true, + planner.enable_nljoin_for_scalar_only:true, + planner.enable_streamagg:true, + planner.enable_type_inference: true, + planner.enable_unionall_distribute:false, + planner.filter.max_selectivity_estimate_factor:1.0, + planner.filter.min_selectivity_estimate_factor:0.0, + planner.force_2phase_aggr : false, + planner.identifier_max_length:1024, + planner.in_subquery_threshold: 20, + planner.join.hash_join_swap_margin_factor:10, + planner.join.row_count_estimate_factor:1.0, + planner.memory.average_field_width: 8, + planner.memory.enable_memory_estimation: false, + planner.memory.hash_agg_table_factor: 1.1d, + planner.memory.hash_join_table_factor: 1.1d, + planner.memory.max_query_memory_per_node: 2147483648, + planner.memory.min_memory_per_buffered_op: 41943040, + planner.memory.non_blocking_operators_memory: 64, + planner.memory_limit:268435456, + planner.nestedloopjoin_factor:100.0, + planner.parser.quoting_identifiers :"`", + planner.partitioner_sender_max_threads:8, + planner.partitioner_sender_set_threads:-1, + planner.partitioner_sender_threads_factor:2, + planner.producer_consumer_queue_size:10, + planner.slice_target: 100000, + planner.store.parquet.rowgroup.filter.pushdown.enabled : true, + planner.store.parquet.rowgroup.filter.pushdown.threshold : 10000, + # Max per node should always be configured as zero and + # it is dynamically computed based on cpu_load_average + planner.width.max_per_node: 0, + planner.width.max_per_query: 1000, + prepare.statement.create_timeout_ms: 10000, + security.admin.user_groups: true, + security.admin.users: true, + store.format: "parquet", + store.hive.optimize_scan_with_native_readers: false, + store.json.all_text_mode: false, + store.json.extended_types: false, + store.json.read_numbers_as_double: false, + store.json.reader.print_skipped_invalid_record_number: false, + store.json.reader.skip_invalid_records: false, + store.json.writer.skip_null_fields: true, + store.json.writer.uglify: false, + store.mongo.all_text_mode: false, + store.mongo.bson.record.reader: true, + store.mongo.read_numbers_as_double: false, + store.parquet.block-size: 536870912, + store.parquet.compression: "none", + store.parquet.dictionary.page-size: 1048576, + store.parquet.enable_dictionary_encoding: false, + store.parquet.page-size: 1048576, + store.parquet.reader.columnreader.async: false, + store.parquet.reader.int96_as_timestamp: false, + store.parquet.reader.pagereader.async: true, + store.parquet.reader.pagereader.bufferedread: true, + store.parquet.reader.pagereader.buffersize: 1048576, + store.parquet.reader.pagereader.enforceTotalSize: false, + store.parquet.reader.pagereader.queuesize: 2, + store.parquet.reader.pagereader.usefadvise: false, + store.parquet.use_new_reader: false, + store.parquet.vector_fill_check_threshold: 10, + store.parquet.vector_fill_threshold: 85, + store.parquet.writer.use_single_fs_block : false, + store.partition.hash_distribute: false, + store.text.estimated_row_size_bytes: 100.0, + web.logs.max_lines: 10000, + window.enable: true, +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java index df0c89f1390..b00d4d62099 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java @@ -80,7 +80,7 @@ protected void testSqlPlan(String sqlCommands) throws Exception { provider.start(); final ScanResult scanResult = ClassPathScanner.fromPrescan(config); final LogicalPlanPersistence logicalPlanPersistence = new LogicalPlanPersistence(config, scanResult); - final SystemOptionManager systemOptions = new SystemOptionManager(logicalPlanPersistence , provider); + final SystemOptionManager systemOptions = new SystemOptionManager(logicalPlanPersistence , provider, config); systemOptions.init(); @SuppressWarnings("resource") final UserSession userSession = UserSession.Builder.newBuilder().withOptionManager(systemOptions).build(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java index 54ae774df6a..8e767e4bcf9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java +++ b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java @@ -41,6 +41,7 @@ import org.apache.drill.exec.server.RemoteServiceSet; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import org.apache.drill.exec.util.VectorUtil; /** @@ -190,7 +191,7 @@ public static OptionValue setupScalarReplacementOption( final OptionManager optionManager = drillbitContext.getOptionManager(); final OptionValue originalOptionValue = optionManager.getOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION); final OptionValue newOptionValue = OptionValue.createString(OptionValue.OptionType.SYSTEM, - ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase()); + ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase(), OptionScope.SYSTEM); optionManager.setOption(newOptionValue); // flush the code cache diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java index dead858fd56..7dd76801abf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java @@ -65,7 +65,7 @@ public void clear(){ public static void setupOptionManager() throws Exception{ final LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(c); provider.start(); - optionManager = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), provider); + optionManager = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), provider,c); optionManager.init(); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java index f4903d8f8e8..7b74a3acad2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java @@ -30,7 +30,7 @@ public static CodeCompiler getTestCompiler(DrillConfig c) throws Exception { DrillConfig config = checkNotNull(c); LogicalPlanPersistence persistence = new LogicalPlanPersistence(config, ClassPathScanner.fromPrescan(config)); LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(config); - SystemOptionManager systemOptionManager = new SystemOptionManager(persistence, provider); + SystemOptionManager systemOptionManager = new SystemOptionManager(persistence, provider, config); return new CodeCompiler(config, systemOptionManager.init()); } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java index 7728aae068c..3c8ca015cb0 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java @@ -29,6 +29,7 @@ import org.apache.drill.exec.rpc.user.UserSession; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import org.apache.drill.exec.server.options.SessionOptionManager; import org.codehaus.commons.compiler.CompileException; import org.junit.Assert; @@ -58,7 +59,7 @@ public static void beforeTestClassTransformation() throws Exception { @Test public void testJaninoClassCompiler() throws Exception { logger.debug("Testing JaninoClassCompiler"); - sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JANINO.name())); + sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JANINO.name(), OptionScope.SESSION)); for (int i = 0; i < ITERATION_COUNT; i++) { compilationInnerClass(false); // Traditional byte-code manipulation compilationInnerClass(true); // Plain-old Java @@ -68,7 +69,8 @@ public void testJaninoClassCompiler() throws Exception { @Test public void testJDKClassCompiler() throws Exception { logger.debug("Testing JDKClassCompiler"); - sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name())); + OptionType type = OptionType.SESSION; + sessionOptions.setOption(OptionValue.createString(type, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name(), OptionScope.SESSION)); for (int i = 0; i < ITERATION_COUNT; i++) { compilationInnerClass(false); // Traditional byte-code manipulation compilationInnerClass(true); // Plain-old Java @@ -80,9 +82,10 @@ public void testCompilationNoDebug() throws CompileException, ClassNotFoundExcep CodeGenerator cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class); ClassSet classSet = new ClassSet(null, cg.getDefinition().getTemplateClassName(), cg.getMaterializedClassName()); String sourceCode = cg.generateAndGet(); - sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name())); + OptionType type = OptionType.SESSION; + sessionOptions.setOption(OptionValue.createString(type, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name(), OptionScope.SESSION)); - sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, false)); + sessionOptions.setOption(OptionValue.createBoolean(type, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, false, OptionScope.SESSION)); @SuppressWarnings("resource") QueryClassLoader loader = new QueryClassLoader(config, sessionOptions); final byte[][] codeWithoutDebug = loader.getClassByteCode(classSet.generated, sourceCode); @@ -92,7 +95,7 @@ public void testCompilationNoDebug() throws CompileException, ClassNotFoundExcep sizeWithoutDebug += bs.length; } - sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, true)); + sessionOptions.setOption(OptionValue.createBoolean(type, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, true, OptionScope.SESSION)); loader = new QueryClassLoader(config, sessionOptions); final byte[][] codeWithDebug = loader.getClassByteCode(classSet.generated, sourceCode); loader.close(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java index 9b507fea33f..1c36c46d649 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java @@ -56,7 +56,7 @@ public static void main(String[] args) throws Exception { check(output); final DrillConfig c = DrillConfig.forClient(); - final SystemOptionManager m = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), new LocalPersistentStoreProvider(c)); + final SystemOptionManager m = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), new LocalPersistentStoreProvider(c), c); m.init(); try (QueryClassLoader ql = new QueryClassLoader(DrillConfig.create(), m)) { ql.injectByteCode("org.apache.drill.Pickle$OutgoingBatch", output); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java index 0d5393ec8da..f9a6358b01e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java @@ -49,7 +49,7 @@ private static boolean checkPrivileges(final String proxyName, final String targ ExecConstants.IMPERSONATION_POLICY_VALIDATOR.validate( OptionValue.createString(OptionValue.OptionType.SYSTEM, ExecConstants.IMPERSONATION_POLICIES_KEY, - IMPERSONATION_POLICIES), null); + IMPERSONATION_POLICIES,OptionValue.OptionScope.SYSTEM), null); try { return InboundImpersonationManager.hasImpersonationPrivileges(proxyName, targetName, IMPERSONATION_POLICIES); } catch (final Exception e) { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java index 82241d746d4..598bdc2ca9d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java @@ -63,6 +63,7 @@ import org.apache.drill.exec.server.options.OptionList; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import org.apache.drill.exec.util.Utilities; import org.apache.drill.exec.work.QueryWorkUnit; import org.junit.AfterClass; @@ -180,20 +181,20 @@ public void testPartitionSenderCostToThreads() throws Exception { final OptionList options = new OptionList(); // try multiple scenarios with different set of options - options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1)); + options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1, OptionScope.SESSION)); testThreadsHelper(hashToRandomExchange, drillbitContext, options, incoming, registry, planReader, planningSet, rootFragment, 1); options.clear(); - options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1)); - options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_max_threads", 10)); + options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1, OptionScope.SESSION)); + options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_max_threads", 10, OptionScope.SESSION)); hashToRandomExchange.setCost(1000); testThreadsHelper(hashToRandomExchange, drillbitContext, options, incoming, registry, planReader, planningSet, rootFragment, 10); options.clear(); - options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1000)); - options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_threads_factor",2)); + options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1000, OptionScope.SESSION)); + options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_threads_factor",2, OptionScope.SESSION)); hashToRandomExchange.setCost(14000); testThreadsHelper(hashToRandomExchange, drillbitContext, options, incoming, registry, planReader, planningSet, rootFragment, 2); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java index 5365773ee3c..39c9ea3d184 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java @@ -32,6 +32,7 @@ import org.apache.drill.exec.rpc.user.UserSession.QueryCountIncrementer; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; /** * Static methods for constructing exception and pause injections for testing purposes. @@ -71,7 +72,7 @@ public static void setControls(final DrillClient drillClient, final String contr public static void setControls(final UserSession session, final String controls) { validateControlsString(controls); final OptionValue opValue = OptionValue.createString(OptionValue.OptionType.SESSION, - DRILLBIT_CONTROL_INJECTIONS, controls); + DRILLBIT_CONTROL_INJECTIONS, controls, OptionScope.SESSION); final OptionManager options = session.getOptions(); try { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java index f3bd63a67cd..9ed352795bc 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java @@ -152,7 +152,7 @@ public void tables() throws Exception { assertEquals(RequestStatus.OK, resp.getStatus()); List tables = resp.getTablesList(); - assertEquals(11, tables.size()); + assertEquals(12, tables.size()); verifyTable("INFORMATION_SCHEMA", "CATALOGS", tables); verifyTable("INFORMATION_SCHEMA", "COLUMNS", tables); @@ -186,7 +186,7 @@ public void tablesWithSystemTableFilter() throws Exception { assertEquals(RequestStatus.OK, resp.getStatus()); List tables = resp.getTablesList(); - assertEquals(11, tables.size()); + assertEquals(12, tables.size()); verifyTable("INFORMATION_SCHEMA", "CATALOGS", tables); verifyTable("INFORMATION_SCHEMA", "COLUMNS", tables); @@ -211,7 +211,7 @@ public void tablesWithTableNameFilter() throws Exception { assertEquals(RequestStatus.OK, resp.getStatus()); List tables = resp.getTablesList(); - assertEquals(4, tables.size()); + assertEquals(5, tables.size()); verifyTable("sys", "boot", tables); verifyTable("sys", "memory", tables); @@ -242,7 +242,7 @@ public void columns() throws Exception { assertEquals(RequestStatus.OK, resp.getStatus()); List columns = resp.getColumnsList(); - assertEquals(71, columns.size()); + assertEquals(76, columns.size()); // too many records to verify the output. } diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java index b305609c147..d6bb3788d61 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java @@ -49,6 +49,7 @@ public RuntimeOption(String key, Object value) { public static final int DEFAULT_ZK_REFRESH = 500; // ms public static final int DEFAULT_SERVER_RPC_THREADS = 10; public static final int DEFAULT_SCAN_THREADS = 8; + public static final String OPTION_DEFAULTS_ROOT = "drill.exec.options."; protected ConfigBuilder configBuilder = new ConfigBuilder(); protected List sessionOptions; @@ -96,6 +97,14 @@ public FixtureBuilder configResource(String configResource) { return this; } + /** + * + */ + public FixtureBuilder setOptionDefault(String key, Object value) { + String option_name = OPTION_DEFAULTS_ROOT + key; + configBuilder().put(option_name, value.toString()); + return this; + } /** * Add an additional boot-time property for the embedded Drillbit. * @param key config property name diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java index 976812c5950..4a63bb57af7 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java @@ -46,6 +46,7 @@ import org.apache.drill.exec.server.options.OptionSet; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.OptionValue.OptionType; +import org.apache.drill.exec.server.options.OptionValue.OptionScope; import org.apache.drill.exec.testing.ExecutionControls; import org.apache.drill.test.rowSet.DirectRowSet; import org.apache.drill.test.rowSet.HyperRowSetImpl; @@ -127,19 +128,19 @@ public void set(String key, int value) { } public void set(String key, long value) { - values.put(key, OptionValue.createLong(OptionType.SYSTEM, key, value)); + values.put(key, OptionValue.createLong(OptionType.SYSTEM, key, value, OptionScope.SYSTEM)); } public void set(String key, boolean value) { - values.put(key, OptionValue.createBoolean(OptionType.SYSTEM, key, value)); + values.put(key, OptionValue.createBoolean(OptionType.SYSTEM, key, value, OptionScope.SYSTEM)); } public void set(String key, double value) { - values.put(key, OptionValue.createDouble(OptionType.SYSTEM, key, value)); + values.put(key, OptionValue.createDouble(OptionType.SYSTEM, key, value, OptionScope.SYSTEM)); } public void set(String key, String value) { - values.put(key, OptionValue.createString(OptionType.SYSTEM, key, value)); + values.put(key, OptionValue.createString(OptionType.SYSTEM, key, value, OptionScope.SYSTEM)); } @Override diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java new file mode 100644 index 00000000000..19f2f06bddd --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java @@ -0,0 +1,209 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.test; + +import org.apache.drill.exec.ExecConstants; +import org.junit.Assert; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +/* + * Tests to test if the linkage between the two config option systems + * i.e., the linkage between boot-config system and system/session options. + * Tests to assert if the config options are read in the order of session , system, boot-config. + * Max width per node is slightly different from other options since it is set to zero by default + * in the config and the option value is computed dynamically everytime if the value is zero + * i.e., if the value is not set in system/session. + * */ + +public class TestConfigLinkage { + + /* Test if session option takes precedence */ + @Test + public void testSessionOption() throws Exception { + FixtureBuilder builder = ClusterFixture.builder().sessionOption(ExecConstants.SLICE_TARGET, 10); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString(); + assertEquals(slice_target, "10"); + } + } + + /* Test if system option takes precedence over the boot option */ + @Test + public void testSystemOption() throws Exception { + FixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 10000); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString(); + assertEquals(slice_target, "10000"); + } + } + + /* Test if config option takes effect if system/session are not set */ + @Test + public void testConfigOption() throws Exception { + FixtureBuilder builder = ClusterFixture.builder() + .setOptionDefault(ExecConstants.SLICE_TARGET, 1000); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString(); + assertEquals(slice_target, "1000"); + } + } + + /* Test if altering system option takes precedence over config option */ + @Test + public void testAlterSystem() throws Exception { + try (ClusterFixture cluster = ClusterFixture.standardCluster(); + ClientFixture client = cluster.clientFixture()) { + client.queryBuilder().sql("ALTER SYSTEM SET `planner.slice_target` = 10000").run(); + String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString(); + assertEquals(slice_target, "10000"); + } + } + + /* Test if altering session option takes precedence over system option */ + @Test + public void testSessionPrecedence() throws Exception { + FixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 100000); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + client.queryBuilder().sql("ALTER SESSION SET `planner.slice_target` = 10000").run(); + String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString(); + assertEquals(slice_target, "10000"); + } + } + + /* Test if setting maxwidth option through config takes effect */ + @Test + public void testMaxWidthPerNodeConfig() throws Exception { + FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.width.max_per_node'").singletonString(); + assertEquals("2", maxWidth); + } + } + + /* Test if setting maxwidth at system level takes precedence */ + @Test + public void testMaxWidthPerNodeSystem() throws Exception { + FixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 3); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.width.max_per_node'").singletonString(); + assertEquals("3", maxWidth); + } + } + + /* Test if setting maxwidth at session level takes precedence */ + @Test + public void testMaxWidthPerNodeSession() throws Exception { + FixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.width.max_per_node'").singletonString(); + assertEquals("2", maxWidth); + } + } + + /* Test if max width is computed correctly using the cpu load average + when the option is not set at either system or session level + */ + @Test + public void testMaxWidthPerNodeDefault() throws Exception { + FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.70); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + long maxWidth = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(0.70, 0); + int availProc = Runtime.getRuntime().availableProcessors(); + long maxWidthPerNode = Math.max(1, Math.min(availProc, Math.round(availProc * 0.70))); + assertEquals(maxWidthPerNode, maxWidth); + } + } + + /* Test if the scope is set during BOOT time and scope is actually BOOT */ + @Test + public void testScope() throws Exception { + FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.SLICE_TARGET, 100000); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + String scope = client.queryBuilder() + .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'") + .singletonString(); + Assert.assertEquals("BOOT",scope); + } + } + + /* Test if the option is set at SYSTEM scope and the scope is actually SYSTEM */ + @Test + public void testScopeSystem() throws Exception { + FixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.SLICE_TARGET, 10000); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + String scope = client.queryBuilder() + .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'") + .singletonString(); + Assert.assertEquals("SYSTEM",scope); + } + } + + /* Test if the option is set at SESSION scope and the scope is actually SESSION */ + @Test + public void testScopeSession() throws Exception { + FixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.SLICE_TARGET, 100000); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + String scope = client.queryBuilder() + .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'") + .singletonString(); + Assert.assertEquals("SESSION",scope); + } + } + + /* Test if the option is altered at SYSTEM scope and the scope is actually SYSTEM */ + @Test + public void testScopeAlterSystem() throws Exception { + FixtureBuilder builder = ClusterFixture.bareBuilder(); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + client.queryBuilder().sql("ALTER SYSTEM set `planner.slice_target`= 10000").run(); + String scope = client.queryBuilder() + .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'") + .singletonString(); + Assert.assertEquals("SYSTEM",scope); + } + } + + /* Test if the option is altered at SESSION scope and the scope is actually SESSION */ + @Test + public void testScopeAlterSession() throws Exception { + FixtureBuilder builder = ClusterFixture.bareBuilder(); + try (ClusterFixture cluster = builder.build(); + ClientFixture client = cluster.clientFixture()) { + client.queryBuilder().sql("ALTER SESSION set `planner.slice_target`= 10000").run(); + String scope = client.queryBuilder() + .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'") + .singletonString(); + Assert.assertEquals("SESSION",scope); + } + } + + +} diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java index b8596503c02..913834b12d5 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java @@ -81,7 +81,7 @@ public void tablesWithConditions() throws Exception{ public ResultSet getResult(Connection c) throws SQLException { return c.getMetaData().getTables("DRILL", "sys", "opt%", new String[]{"SYSTEM_TABLE", "SYSTEM_VIEW"}); } - }, 1); + }, 2); } @Test @@ -101,6 +101,6 @@ public void columnsWithConditions() throws Exception{ public ResultSet getResult(Connection c) throws SQLException { return c.getMetaData().getColumns("DRILL", "sys", "opt%", "%ame"); } - }, 1); + }, 2); } }