diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index df311bf216c..c496b9e5028 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -19,6 +19,8 @@ # This file tells Drill to consider this module when class path scanning. # This file can also include any supplementary configuration information. # This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. +# Please keep the lists that follow sorted (while respecting existing +# groupings and comment lines). drill { classpath.scanning { @@ -42,13 +44,13 @@ drill { ], packages: ${?drill.classpath.scanning.packages} [ + org.apache.drill.exec.coord.zk, org.apache.drill.exec.expr, org.apache.drill.exec.physical, - org.apache.drill.exec.store, - org.apache.drill.exec.rpc.user.security, org.apache.drill.exec.rpc.security, + org.apache.drill.exec.rpc.user.security, org.apache.drill.exec.server.rest.auth, - org.apache.drill.exec.coord.zk, + org.apache.drill.exec.store, org.apache.drill.exec.store.security ], @@ -542,70 +544,82 @@ drill.exec.options: { # enabled by setting this flag to true. By default it's set to false such that # query will fail if there is not enough memory drill.exec.hashagg.fallback.enabled: false, + drill.exec.http.rest.errors.verbose: false, + drill.exec.implicit_casts.joins.enabled: false, + drill.exec.memory.operator.output_batch_size : 16777216, # 16 MB + drill.exec.memory.operator.output_batch_size_avail_mem_factor : 0.1, + drill.exec.stats.logging.batch_size: false, + drill.exec.stats.logging.fine_grained.batch_size: false, + drill.exec.stats.logging.enabled_operators: all, drill.exec.storage.file.partition.column.label: "dir", drill.exec.storage.implicit.filename.column.label: "filename", drill.exec.storage.implicit.filepath.column.label: "filepath", drill.exec.storage.implicit.fqn.column.label: "fqn", - drill.exec.storage.implicit.suffix.column.label: "suffix", - drill.exec.storage.implicit.row_group_index.column.label: "rgi", - drill.exec.storage.implicit.row_group_start.column.label: "rgs", - drill.exec.storage.implicit.row_group_length.column.label: "rgl", drill.exec.storage.implicit.last_modified_time.column.label: "lmt", drill.exec.storage.implicit.project_metadata.column.label: "$project_metadata$", - drill.exec.implicit_casts.joins.enabled: false, + drill.exec.storage.implicit.row_group_index.column.label: "rgi", + drill.exec.storage.implicit.row_group_length.column.label: "rgl", + drill.exec.storage.implicit.row_group_start.column.label: "rgs", + drill.exec.storage.implicit.suffix.column.label: "suffix", drill.exec.testing.controls: "{}", - drill.exec.memory.operator.output_batch_size : 16777216, # 16 MB - drill.exec.memory.operator.output_batch_size_avail_mem_factor : 0.1, exec.bulk_load_table_list.bulk_size: 1000, + exec.enable_aliases: true, exec.enable_bulk_load_table_list: false, + exec.enable_dynamic_fc: false, exec.enable_union_type: false, - exec.except_add_agg_below: false, exec.errors.verbose: false, - drill.exec.http.rest.errors.verbose: false, - exec.hashjoin.mem_limit: 0, - exec.hashjoin.hash_table_calc_type: "LEAN", - exec.hashjoin.safety_factor: 1.0, - exec.hashjoin.fragmentation_factor: 1.33, - exec.hashjoin.hash_double_factor: 2.0, - exec.hashjoin.num_partitions: 32, - exec.hashjoin.num_rows_in_batch: 1024, - exec.hashjoin.max_batches_in_memory: 0, - exec.hashjoin.enable.runtime_filter: false, - exec.hashjoin.bloom_filter.fpp: 0.75, - exec.hashjoin.bloom_filter.max.size: 33554432, #32 MB - exec.hashjoin.runtime_filter.waiting.enable: true, - exec.hashjoin.runtime_filter.max.waiting.time: 300, #400 ms + exec.except_add_agg_below: false, + + exec.hashagg.max_batches_in_memory: 65536, exec.hashagg.mem_limit: 0, exec.hashagg.min_batches_per_partition: 2, exec.hashagg.num_partitions: 32, exec.hashagg.num_rows_in_batch: 128, - exec.hashagg.max_batches_in_memory: 65536, exec.hashagg.use_memory_prediction: true, + + exec.hashjoin.bloom_filter.fpp: 0.75, + exec.hashjoin.bloom_filter.max.size: 33554432, #32 MB + exec.hashjoin.enable.runtime_filter: false, + exec.hashjoin.fragmentation_factor: 1.33, + exec.hashjoin.hash_double_factor: 2.0, + exec.hashjoin.hash_table_calc_type: "LEAN", + exec.hashjoin.max_batches_in_memory: 0, + exec.hashjoin.mem_limit: 0, + exec.hashjoin.num_partitions: 32, + exec.hashjoin.num_rows_in_batch: 1024, + exec.hashjoin.runtime_filter.max.waiting.time: 300, #400 ms + exec.hashjoin.runtime_filter.waiting.enable: true, + exec.hashjoin.safety_factor: 1.0, + exec.impersonation.inbound_policies: "[]", - exec.java.compiler.exp_in_method_size: 50, - exec.java_compiler: "DEFAULT", + exec.java_compiler_debug: true, + exec.java_compiler: "DEFAULT", + exec.java.compiler.exp_in_method_size: 50, exec.java_compiler_janino_maxsize: 262144, + exec.max_hash_table_size: 1073741824, exec.min_hash_table_size: 65536, exec.partition.mem_throttle: 0, exec.persistent_table.umask: "002", - exec.query.progress.update: true, + + exec.query_profile.alter_session.skip: true, exec.query_profile.debug_mode: false, exec.query_profile.save: true, - exec.query_profile.alter_session.skip: true, - exec.enable_dynamic_fc: false, - exec.enable_aliases: true, - exec.queue.enable: false, + + exec.query.progress.update: true, + # Default queue values for an 8 GB direct memory default # Drill install. Users are expected to adjust these based # on load and available resources. + exec.queue.enable: false, exec.queue.large: 2, + exec.queue.memory_ratio: 10.0, + exec.queue.memory_reserve_ratio: 0.2, exec.queue.small: 4, exec.queue.threshold: 30000000, exec.queue.timeout_millis: 300000, - exec.queue.memory_ratio: 10.0, - exec.queue.memory_reserve_ratio: 0.2, + # Deprecated exec.sort.disable_managed : false, # Deprecated @@ -616,11 +630,9 @@ drill.exec.options: { exec.storage.skip_runtime_rowgroup_pruning: false, exec.udf.enable_dynamic_support: true, exec.udf.use_dynamic: true, - drill.exec.stats.logging.batch_size: false, - drill.exec.stats.logging.fine_grained.batch_size: false, - drill.exec.stats.logging.enabled_operators: all, new_view_default_permissions: 700, org.apache.drill.exec.compile.ClassTransformer.scalar_replacement: "try", + planner.add_producer_consumer: false, planner.affinity_factor: 1.2, planner.broadcast_factor: 1.0, @@ -631,26 +643,24 @@ drill.exec.options: { planner.enable_constant_folding: true, planner.enable_decimal_data_type: true, planner.enable_demux_exchange: false, - planner.enable_hash_single_key: true, + planner.enable_file_listing_limit0_optimization: false, planner.enable_hashagg: true, - planner.enable_hashjoin: true, - planner.enable_semijoin: true, planner.enable_hashjoin_swap: true, + planner.enable_hashjoin: true, + planner.enable_hash_single_key: true, planner.enable_hep_opt: true, planner.enable_hep_partition_pruning: true, - planner.enable_rowkeyjoin_conversion: true, - planner.rowkeyjoin_conversion_using_hashjoin: false, - planner.rowkeyjoin_conversion_selectivity_threshold: 0.01, planner.enable_join_optimization: true, - planner.enable_limit0_optimization: true, planner.enable_limit0_on_scan: true, - planner.enable_file_listing_limit0_optimization: false, + planner.enable_limit0_optimization: true, planner.enable_mergejoin: true, planner.enable_multiphase_agg: true, planner.enable_mux_exchange: true, - planner.enable_ordered_mux_exchange: true, planner.enable_nestedloopjoin: true, planner.enable_nljoin_for_scalar_only: true, + planner.enable_ordered_mux_exchange: true, + planner.enable_rowkeyjoin_conversion: true, + planner.enable_semijoin: true, planner.enable_streamagg: true, planner.enable_topn: true, planner.enable_type_inference: true, @@ -667,132 +677,142 @@ drill.exec.options: { planner.memory.enable_memory_estimation: false, planner.memory.hash_agg_table_factor: 1.1d, planner.memory.hash_join_table_factor: 1.1d, + planner.memory_limit: 268435456, planner.memory.max_query_memory_per_node: 2147483648, # 2 GB - planner.memory.percent_per_query: 0.05, # 5% planner.memory.min_memory_per_buffered_op: 41943040, # 40 MB planner.memory.non_blocking_operators_memory: 64, - planner.memory_limit: 268435456, + planner.memory.percent_per_query: 0.05, # 5% planner.nestedloopjoin_factor: 100.0, planner.parser.quoting_identifiers: "`", planner.partitioner_sender_max_threads: 8, planner.partitioner_sender_set_threads: -1, planner.partitioner_sender_threads_factor: 2, planner.producer_consumer_queue_size: 10, + planner.rowkeyjoin_conversion_selectivity_threshold: 0.01, + planner.rowkeyjoin_conversion_using_hashjoin: false, planner.slice_target: 100000, - planner.statistics.use: false, planner.statistics.multicol_ndv_adjustment_factor: 1.0, + planner.statistics.use: false, planner.store.parquet.rowgroup.filter.pushdown.enabled: true, planner.store.parquet.rowgroup.filter.pushdown.threshold: 10000, # Max per node should always be configured as zero and # it is dynamically computed based on cpu_load_average planner.width.max_per_node: 0, planner.width.max_per_query: 1000, + prepare.statement.create_timeout_ms: 30000, security.admin.user_groups: "%drill_process_user_groups%", security.admin.users: "%drill_process_user%", + storage.list_files_recursively: false, + storage.plugin_auto_disable: false, + storage.plugin_retry_attempt_delay: 2000, + storage.plugin_retry_attempts: 1, store.format: "parquet", - store.hive.optimize_scan_with_native_readers: false, - store.hive.parquet.optimize_scan_with_native_reader: false, # Properties values should NOT be set in double-quotes or any other quotes. # Property name and value should be separated by =. # Properties should be separated by new line (\n). store.hive.conf.properties: "", - store.json.enable_v2_reader: true, + store.hive.optimize_scan_with_native_readers: false, + store.hive.parquet.optimize_scan_with_native_reader: false, store.json.all_text_mode: false, - store.json.writer.allow_nan_inf: true, - store.json.reader.allow_nan_inf: true, - store.json.reader.allow_escape_any_char: false, + store.json.enable_v2_reader: true, store.json.extended_types: false, - store.json.read_numbers_as_double: false, + store.json.reader.allow_escape_any_char: false, + store.json.reader.allow_nan_inf: true, store.json.reader.print_skipped_invalid_record_number: false, store.json.reader.skip_invalid_records: false, + store.json.read_numbers_as_double: false, + store.json.writer.allow_nan_inf: true, store.json.writer.skip_null_fields: true, store.json.writer.uglify: false, + store.kafka.all_text_mode: false, + store.kafka.poll.timeout: 200, + store.kafka.reader.allow_escape_any_char: false, + store.kafka.reader.allow_nan_inf: true, + store.kafka.reader.skip_invalid_records: false, + store.kafka.read_numbers_as_double: false, + store.kafka.record.reader: "org.apache.drill.exec.store.kafka.decoders.JsonMessageReader", store.mongo.all_text_mode: false, store.mongo.bson.record.reader: true, store.mongo.read_numbers_as_double: false, store.parquet.block-size: 536870912, + + store.parquet.complex.batch.num_records: 4000, store.parquet.compression: "none", store.parquet.dictionary.page-size: 1048576, store.parquet.enable_dictionary_encoding: false, + # Using common operators batch configuration unless the Parquet specific + # configuration is used + store.parquet.flat.batch.memory_size: 0, + store.parquet.flat.batch.num_records: 32767, + store.parquet.flat.reader.bulk: true, store.parquet.page-size: 1048576, store.parquet.reader.columnreader.async: false, + store.parquet.reader.enable_map_support: true, store.parquet.reader.int96_as_timestamp: false, - store.parquet.reader.strings_signed_min_max: "", store.parquet.reader.pagereader.async: true, store.parquet.reader.pagereader.bufferedread: true, store.parquet.reader.pagereader.buffersize: 1048576, store.parquet.reader.pagereader.enforceTotalSize: false, store.parquet.reader.pagereader.queuesize: 2, store.parquet.reader.pagereader.usefadvise: false, - store.parquet.reader.enable_map_support: true, + store.parquet.reader.strings_signed_min_max: "", store.parquet.use_new_reader: false, store.parquet.vector_fill_check_threshold: 10, store.parquet.vector_fill_threshold: 85, - store.parquet.writer.use_primitive_types_for_decimals: true, + store.parquet.writer.format_version: "v1", store.parquet.writer.logical_type_for_decimals: "fixed_len_byte_array", + store.parquet.writer.use_primitive_types_for_decimals: true, store.parquet.writer.use_single_fs_block: false, - store.parquet.writer.format_version: "v1", - store.parquet.flat.reader.bulk: true, - store.parquet.flat.batch.num_records: 32767, - store.parquet.complex.batch.num_records: 4000, - # Using common operators batch configuration unless the Parquet specific - # configuration is used - store.parquet.flat.batch.memory_size: 0, - store.table.use_schema_file: false, + store.partition.hash_distribute: false, + store.table.use_schema_file: false, store.text.estimated_row_size_bytes: 100.0, store.text.writer.add_header: true, store.text.writer.force_quotes: false, - store.kafka.all_text_mode: false, - store.kafka.read_numbers_as_double: false, - store.kafka.record.reader: "org.apache.drill.exec.store.kafka.decoders.JsonMessageReader", - store.kafka.poll.timeout: 200, - store.kafka.reader.skip_invalid_records: false, - store.kafka.reader.allow_nan_inf: true, - store.kafka.reader.allow_escape_any_char: false, - web.logs.max_lines: 10000, - web.display_format.timestamp: "", web.display_format.date: "", web.display_format.time: "", + web.display_format.timestamp: "", + web.logs.max_lines: 10000, window.enable: true, - storage.list_files_recursively: false, - storage.plugin_retry_attempts: 1, - storage.plugin_retry_attempt_delay: 2000, - storage.plugin_auto_disable: false, + # ============ index plan related options ============== - planner.use_simple_optimizer: false, + planner.disable_full_table_scan: false, planner.enable_index_planning: true, planner.enable_statistics: true, - planner.disable_full_table_scan: false, - planner.index.max_chosen_indexes_per_table: 5, - planner.index.force_sort_noncovering: false, - planner.index.use_hashjoin_noncovering: false, planner.index.covering_selectivity_threshold: 1.0, + planner.index.force_sort_noncovering: false, + planner.index.max_chosen_indexes_per_table: 5, + planner.index.max_indexes_to_intersect: 5, planner.index.noncovering_selectivity_threshold: 0.025, + planner.index.prefer_intersect_plans: false, planner.index.rowkeyjoin_cost_factor: 0.1, planner.index.statistics_rowcount_scaling_factor: 0.1, - planner.index.prefer_intersect_plans: false, - planner.index.max_indexes_to_intersect: 5, - exec.query.rowkeyjoin_batchsize: 128, - exec.query.return_result_set_for_ddl: true, + planner.index.use_hashjoin_noncovering: false, + planner.use_simple_optimizer: false, + exec.query.max_rows: 0, - exec.return_result_set_for_ddl: true, - exec.statistics.ndv_accuracy: 20, - exec.statistics.ndv_extrapolation_bf_elements: 1000000, - exec.statistics.ndv_extrapolation_bf_fpprobability: 10, - exec.statistics.deterministic_sampling: false, exec.query.return_result_set_for_ddl: true, + exec.query.return_result_set_for_ddl: true, + exec.query.rowkeyjoin_batchsize: 128, + exec.return_result_set_for_ddl: true, + # ========= rm related options =========== exec.rm.queryTags: "", exec.rm.queues.wait_for_preferred_nodes: true, + + exec.statistics.deterministic_sampling: false, + exec.statistics.ndv_accuracy: 20, + exec.statistics.ndv_extrapolation_bf_elements: 1000000, + exec.statistics.ndv_extrapolation_bf_fpprobability: 10, exec.statistics.tdigest_compression: 100, + # ========= Metastore related options =========== metastore.enabled: false, - metastore.retrieval.retry_attempts: 5 + metastore.metadata.ctas.auto-collect: "NONE", + metastore.metadata.fallback_to_file_metadata: true metastore.metadata.store.depth_level: "ALL", metastore.metadata.use_schema: true, metastore.metadata.use_statistics: true, - metastore.metadata.ctas.auto-collect: "NONE", - metastore.metadata.fallback_to_file_metadata: true + metastore.retrieval.retry_attempts: 5 } diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml index 79e799a50eb..573853b600e 100644 --- a/exec/jdbc-all/pom.xml +++ b/exec/jdbc-all/pom.xml @@ -882,7 +882,7 @@ hadoop-2 - 54400000 + 54500000 diff --git a/pom.xml b/pom.xml index 83fb666dc21..70cb66b2f61 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ 1.23.0 3.1.11 1.12.0 - 2.14.3 + 2.16.1 3.5.10 1.0.0 3.0.5