Merge pull request #9726 from duckdb/feature

Merge Feature Into Main

.github/config/uncovered_files.csv

@@ -58,7 +58,7 @@ common/serializer/buffered_file_reader.cpp	11
 common/serializer/buffered_file_writer.cpp	2
 common/serializer/binary_serializer.cpp	11
 common/serializer/memory_stream.cpp	5
-common/sort/comparators.cpp	72
+common/sort/comparators.cpp	103
 common/sort/merge_sorter.cpp	100
 common/sort/partition_state.cpp	44
 common/sort/sort_state.cpp	10
@@ -91,7 +91,7 @@ common/types/row/row_layout.cpp	4
 common/types/row/tuple_data_allocator.cpp	23
 common/types/row/tuple_data_collection.cpp	76
 common/types/row/tuple_data_iterator.cpp	5
-common/types/row/tuple_data_scatter_gather.cpp	17
+common/types/row/tuple_data_scatter_gather.cpp	19
 common/types/row/tuple_data_segment.cpp	10
 common/types/string_heap.cpp	11
 common/types/time.cpp	25
@@ -100,7 +100,7 @@ common/types/uuid.cpp	3
 common/types/validity_mask.cpp	13
 common/types/value.cpp	180
 common/types/vector.cpp	153
-common/value_operations/comparison_operations.cpp	32
+common/value_operations/comparison_operations.cpp	33
 common/vector_operations/generators.cpp	31
 common/vector_operations/is_distinct_from.cpp	49
 common/vector_operations/null_operations.cpp	15
@@ -114,7 +114,7 @@ core_functions/aggregate/distributive/bitagg.cpp	8
 core_functions/aggregate/distributive/bitstring_agg.cpp	6
 core_functions/aggregate/distributive/entropy.cpp	9
 core_functions/aggregate/distributive/kurtosis.cpp	3
-core_functions/aggregate/distributive/minmax.cpp	2
+core_functions/aggregate/distributive/minmax.cpp	3
 core_functions/aggregate/distributive/skew.cpp	2
 core_functions/aggregate/distributive/sum.cpp	10
 core_functions/aggregate/holistic/approximate_quantile.cpp	29
@@ -123,6 +123,9 @@ core_functions/aggregate/holistic/quantile.cpp	14
 core_functions/aggregate/holistic/reservoir_quantile.cpp	39
 core_functions/aggregate/nested/histogram.cpp	1
 core_functions/aggregate/nested/list.cpp	4
+core_functions/scalar/array/array_value.cpp	2
+core_functions/scalar/string/length.cpp	2
+core_functions/lambda_functions.cpp	1
 core_functions/scalar/bit/bitstring.cpp	3
 core_functions/scalar/date/date_diff.cpp	124
 core_functions/scalar/date/date_part.cpp	17
@@ -136,7 +139,7 @@ core_functions/scalar/generic/least.cpp	2
 core_functions/scalar/generic/system_functions.cpp	4
 core_functions/scalar/list/array_slice.cpp	3
 core_functions/scalar/list/flatten.cpp	6
-core_functions/scalar/list/list_aggregates.cpp	2
+core_functions/scalar/list/list_aggregates.cpp	6
 core_functions/scalar/list/list_lambdas.cpp	2
 core_functions/scalar/list/list_sort.cpp	7
 core_functions/scalar/list/range.cpp	6
@@ -255,7 +258,7 @@ function/cast/union_casts.cpp	4
 function/cast/uuid_casts.cpp	2
 function/cast/union/from_struct.cpp	0
 function/cast/vector_cast_helpers.cpp	9
-function/cast_rules.cpp	18
+function/cast_rules.cpp	19
 function/function.cpp	8
 function/function_binder.cpp	18
 function/function_set.cpp	8
@@ -270,6 +273,7 @@ function/scalar/operators/arithmetic.cpp	23
 function/scalar/operators/multiply.cpp	3
 function/scalar/operators/subtract.cpp	5
 function/scalar/strftime_format.cpp	31
+function/scalar/string/length.cpp	2
 function/scalar/string/regexp.cpp	5
 function/scalar/string/regexp/regexp_extract_all.cpp	5
 function/scalar/string/substring.cpp	8
@@ -391,6 +395,7 @@ include/duckdb/common/types/string_type.hpp	5
 include/duckdb/common/types/uuid.hpp	4
 include/duckdb/common/types/validity_mask.hpp	5
 include/duckdb/common/types/value.hpp	6
+include/duckdb/common/types/vector_buffer.hpp	1
 include/duckdb/common/vector_operations/aggregate_executor.hpp	17
 include/duckdb/common/vector_operations/binary_executor.hpp	15
 include/duckdb/common/vector_operations/generic_executor.hpp	9
@@ -581,6 +586,7 @@ parser/expression/case_expression.cpp	3
 parser/expression/collate_expression.cpp	3
 parser/expression/default_expression.cpp	8
 parser/expression/function_expression.cpp	2
+parser/expression/lambdaref_expression.cpp	3
 parser/expression/star_expression.cpp	5
 parser/expression/subquery_expression.cpp	5
 parser/expression/window_expression.cpp	8
@@ -707,7 +713,7 @@ planner/expression/bound_lambdaref_expression.cpp	38
 planner/expression/bound_reference_expression.cpp	12
 planner/expression/bound_subquery_expression.cpp	5
 planner/expression/bound_window_expression.cpp	7
-planner/expression_binder.cpp	7
+planner/expression_binder.cpp	9
 planner/expression_binder/aggregate_binder.cpp	3
 planner/expression_binder/alter_binder.cpp	3
 planner/expression_binder/base_select_binder.cpp	3
@@ -794,6 +800,7 @@ storage/standard_buffer_manager.cpp	128
 storage/statistics/base_statistics.cpp	48
 storage/statistics/column_statistics.cpp	8
 storage/statistics/distinct_statistics.cpp	12
+storage/statistics/array_stats.cpp	1
 storage/statistics/list_stats.cpp	13
 storage/statistics/numeric_stats.cpp	117
 storage/statistics/numeric_stats_union.cpp	3
@@ -812,6 +819,7 @@ storage/table/column_data.cpp	16
 storage/table/column_data_checkpointer.cpp	4
 storage/table/column_segment.cpp	62
 storage/table/list_column_data.cpp	28
+storage/table/array_column_data.cpp	25
 storage/table/row_group.cpp	54
 storage/table/row_group_collection.cpp	7
 storage/table/row_version_manager.cpp	6

.github/regression/tpcds.csv

@@ -82,6 +82,7 @@ benchmark/tpcds/sf1/q81.benchmark
 benchmark/tpcds/sf1/q82.benchmark
 benchmark/tpcds/sf1/q83.benchmark
 benchmark/tpcds/sf1/q84.benchmark
+benchmark/tpcds/sf1/q85.benchmark
 benchmark/tpcds/sf1/q86.benchmark
 benchmark/tpcds/sf1/q87.benchmark
 benchmark/tpcds/sf1/q88.benchmark

CMakeLists.txt

@@ -483,6 +483,7 @@ include_directories(third_party/fsst)
 include_directories(third_party/fmt/include)
 include_directories(third_party/hyperloglog)
 include_directories(third_party/fastpforlib)
+include_directories(third_party/skiplist)
 include_directories(third_party/fast_float)
 include_directories(third_party/re2)
 include_directories(third_party/miniz)

benchmark/micro/json/inconsistent_schema.benchmark

@@ -0,0 +1,14 @@
+# name: benchmark/micro/json/inconsistent_schema.benchmark
+# description: Read JSON file with inconsistent schema
+# group: [json]
+
+name JSON inconsistent schema
+group json
+
+require json
+
+run
+CREATE OR REPLACE TABLE test AS SELECT * FROM 'data/json/5000.jsonl'
+
+result I
+5000

benchmark/micro/lambdas/list_filter_where.benchmark

@@ -0,0 +1,13 @@
+# name: benchmark/micro/lambdas/list_filter_where.benchmark
+# description: Simulate a where-function with lambda functions on 100K values
+# group: [lambdas]
+
+name Benchmark list_filter
+group lambdas
+
+load
+CREATE TABLE tbl (l INTEGER[], where_l BOOLEAN[]);
+INSERT INTO tbl VALUES (range(100000), [x % 13 != 0 for x in range(100000)]);
+
+run
+SELECT list_filter(l, (x, x_i) -> where_l[x_i]) FROM tbl;

benchmark/micro/lambdas/list_transform_select.benchmark

@@ -0,0 +1,13 @@
+# name: benchmark/micro/lambdas/list_transform_select.benchmark
+# description: Simulate a select-function with lambda functions on 100K values
+# group: [lambdas]
+
+name Benchmark list_transform
+group lambdas
+
+load
+CREATE TABLE tbl (l INTEGER[], sel INTEGER[]);
+INSERT INTO tbl VALUES (range(100000), range(100000));
+
+run
+SELECT list_transform(sel, x -> l[x]) FROM tbl;

benchmark/micro/lambdas/list_transform_zip.benchmark

@@ -0,0 +1,13 @@
+# name: benchmark/micro/lambdas/list_transform_zip.benchmark
+# description: Simulate a zip-function with lambda functions on 100K values
+# group: [lambdas]
+
+name Benchmark list_transform
+group lambdas
+
+load
+CREATE TABLE tbl (l INTEGER[], zip VARCHAR[]);
+INSERT INTO tbl VALUES (range(100000), [x::VARCHAR || (x + 1)::VARCHAR for x in range(100000)]);
+
+run
+SELECT list_transform(l, (x, x_i) -> (x, zip[x_i])) FROM tbl;