diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index c0734450874b..1b891ab83d93 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -32,14 +32,14 @@ simd = ["datafusion/simd"] snmalloc = ["snmalloc-rs"] [dependencies] -arrow = "24.0.0" +arrow = "25.0.0" datafusion = { path = "../datafusion/core" } env_logger = "0.9" futures = "0.3" mimalloc = { version = "0.1", optional = true, default-features = false } num_cpus = "1.13.0" object_store = "0.5.0" -parquet = "24.0.0" +parquet = "25.0.0" rand = "0.8.4" serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.78" diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 29ddffd6ce9a..364c412fb02d 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -79,9 +79,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68391300d5237f6725f0f869ae7cb65d45fcf8a6d18f6ceecd328fb803bef93" +checksum = "76312eb67808c67341f4234861c4fcd2f9868f55e88fa2186ab3b357a6c5830b" dependencies = [ "ahash 0.8.0", "arrow-array", @@ -107,9 +107,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0bb00c5862b5eea683812083c495bef01a9a5149da46ad2f4c0e4aa8800f64d" +checksum = "69dd2c257fa76de0bcc63cabe8c81d34c46ef6fa7651e3e497922c3c9878bd67" dependencies = [ "ahash 0.8.0", "arrow-buffer", @@ -123,18 +123,19 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e594d0fe0026a8bc2459bdc5ac9623e5fb666724a715e0acbc96ba30c5d4cc7" +checksum = "af963e71bdbbf928231d521083ddc8e8068cf5c8d45d4edcfeaf7eb5cdd779a9" dependencies = [ "half", + "num", ] [[package]] name = "arrow-data" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8500df05060d86fdc53e9b5cb32e51bfeaacc040fdeced3eb99ac0d59200ff45" +checksum = "52554ffff560c366d7210c2621a3cf1dc408f9969a0c7688a3ba0a62248a945d" dependencies = [ "arrow-buffer", "arrow-schema", @@ -144,9 +145,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d1fef01f25e1452c86fa6887f078de8e0aaeeb828370feab205944cfc30e27" +checksum = "1a5518f2bd7775057391f88257627cbb760ba3e1c2f2444a005ba79158624654" [[package]] name = "async-compression" @@ -1619,9 +1620,9 @@ dependencies = [ [[package]] name = "parquet" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fd590f0672998df84503d1bcbebc69732583d03cc3495c7dd8d3e5a1d8437f" +checksum = "f7758803135c32b243e52832473fc8f7c768a0a170b0851fb1bb37904c6b3550" dependencies = [ "ahash 0.8.0", "arrow", diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 61328a8bfd74..7367f2dc0aeb 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -29,7 +29,7 @@ rust-version = "1.62" readme = "README.md" [dependencies] -arrow = "24.0.0" +arrow = "25.0.0" clap = { version = "3", features = ["derive", "cargo"] } datafusion = { path = "../datafusion/core", version = "13.0.0" } dirs = "4.0.0" diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index 2530f890f683..9abf76b253a6 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -34,8 +34,8 @@ path = "examples/avro_sql.rs" required-features = ["datafusion/avro"] [dev-dependencies] -arrow = "24.0.0" -arrow-flight = "24.0.0" +arrow = "25.0.0" +arrow-flight = "25.0.0" async-trait = "0.1.41" datafusion = { path = "../datafusion/core" } datafusion-common = { path = "../datafusion/common" } diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 382f66e5dd98..c8ed7a034ccf 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -40,10 +40,10 @@ pyarrow = ["pyo3", "arrow/pyarrow"] [dependencies] apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true } -arrow = { version = "24.0.0", default-features = false } +arrow = { version = "25.0.0", default-features = false } cranelift-module = { version = "0.88.0", optional = true } object_store = { version = "0.5.0", default-features = false, optional = true } ordered-float = "3.0" -parquet = { version = "24.0.0", default-features = false, optional = true } +parquet = { version = "25.0.0", default-features = false, optional = true } pyo3 = { version = "0.17.1", optional = true } sqlparser = "0.25" diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs index c3f91dd9b1d1..680bd3f66fe0 100644 --- a/datafusion/common/src/scalar.rs +++ b/datafusion/common/src/scalar.rs @@ -34,7 +34,6 @@ use arrow::{ TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type, DECIMAL128_MAX_PRECISION, }, - util::decimal::Decimal128, }; use ordered_float::OrderedFloat; @@ -1696,7 +1695,7 @@ impl ScalarValue { if array.is_null(index) { ScalarValue::Decimal128(None, precision, scale) } else { - ScalarValue::Decimal128(Some(array.value(index).as_i128()), precision, scale) + ScalarValue::Decimal128(Some(array.value(index)), precision, scale) } } @@ -1881,11 +1880,7 @@ impl ScalarValue { } match value { None => array.is_null(index), - Some(v) => { - !array.is_null(index) - && array.value(index) - == Decimal128::new(precision, scale, &v.to_le_bytes()) - } + Some(v) => !array.is_null(index) && array.value(index) == *v, } } @@ -2501,15 +2496,15 @@ mod tests { let array = array.as_any().downcast_ref::().unwrap(); assert_eq!(1, array.len()); assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone()); - assert_eq!(123i128, array.value(0).as_i128()); + assert_eq!(123i128, array.value(0)); // decimal scalar to array with size let array = decimal_value.to_array_of_size(10); let array_decimal = array.as_any().downcast_ref::().unwrap(); assert_eq!(10, array.len()); assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone()); - assert_eq!(123i128, array_decimal.value(0).as_i128()); - assert_eq!(123i128, array_decimal.value(9).as_i128()); + assert_eq!(123i128, array_decimal.value(0)); + assert_eq!(123i128, array_decimal.value(9)); // test eq array assert!(decimal_value.eq_array(&array, 1)); assert!(decimal_value.eq_array(&array, 5)); diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 44a5cbe1afb4..6512db0ae922 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -56,7 +56,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion [dependencies] ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } apache-avro = { version = "0.14", optional = true } -arrow = { version = "24.0.0", features = ["prettyprint"] } +arrow = { version = "25.0.0", features = ["prettyprint"] } async-compression = { version = "0.3.14", features = ["bzip2", "gzip", "futures-io", "tokio"] } async-trait = "0.1.41" bytes = "1.1" @@ -81,7 +81,7 @@ num_cpus = "1.13.0" object_store = "0.5.0" ordered-float = "3.0" parking_lot = "0.12" -parquet = { version = "24.0.0", features = ["arrow", "async"] } +parquet = { version = "25.0.0", features = ["arrow", "async"] } paste = "^1.0" percent-encoding = "2.2.0" pin-project-lite = "^0.2.7" @@ -98,7 +98,7 @@ url = "2.2" uuid = { version = "1.0", features = ["v4"] } [dev-dependencies] -arrow = { version = "24.0.0", features = ["prettyprint", "dyn_cmp_dict"] } +arrow = { version = "25.0.0", features = ["prettyprint", "dyn_cmp_dict"] } async-trait = "0.1.53" criterion = "0.4" csv = "1.1.6" diff --git a/datafusion/core/fuzz-utils/Cargo.toml b/datafusion/core/fuzz-utils/Cargo.toml index 3f1f2ed765a7..ba876d4ffed0 100644 --- a/datafusion/core/fuzz-utils/Cargo.toml +++ b/datafusion/core/fuzz-utils/Cargo.toml @@ -23,6 +23,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -arrow = { version = "24.0.0", features = ["prettyprint"] } +arrow = { version = "25.0.0", features = ["prettyprint"] } env_logger = "0.9.0" rand = "0.8" diff --git a/datafusion/core/src/physical_plan/hash_utils.rs b/datafusion/core/src/physical_plan/hash_utils.rs index 61e72b07c551..899008aa44da 100644 --- a/datafusion/core/src/physical_plan/hash_utils.rs +++ b/datafusion/core/src/physical_plan/hash_utils.rs @@ -62,29 +62,23 @@ fn hash_decimal128<'a>( if array.null_count() == 0 { if mul_col { for (i, hash) in hashes_buffer.iter_mut().enumerate() { - *hash = combine_hashes( - random_state.hash_one(&array.value(i).as_i128()), - *hash, - ); + *hash = combine_hashes(random_state.hash_one(&array.value(i)), *hash); } } else { for (i, hash) in hashes_buffer.iter_mut().enumerate() { - *hash = random_state.hash_one(&array.value(i).as_i128()); + *hash = random_state.hash_one(&array.value(i)); } } } else if mul_col { for (i, hash) in hashes_buffer.iter_mut().enumerate() { if !array.is_null(i) { - *hash = combine_hashes( - random_state.hash_one(&array.value(i).as_i128()), - *hash, - ); + *hash = combine_hashes(random_state.hash_one(&array.value(i)), *hash); } } } else { for (i, hash) in hashes_buffer.iter_mut().enumerate() { if !array.is_null(i) { - *hash = random_state.hash_one(&array.value(i).as_i128()); + *hash = random_state.hash_one(&array.value(i)); } } } diff --git a/datafusion/core/src/test/mod.rs b/datafusion/core/src/test/mod.rs index bb884e142b05..d6e2c05fc4ef 100644 --- a/datafusion/core/src/test/mod.rs +++ b/datafusion/core/src/test/mod.rs @@ -242,14 +242,17 @@ pub fn table_with_decimal() -> Arc { } fn make_decimal() -> RecordBatch { - let mut decimal_builder = Decimal128Builder::with_capacity(20, 10, 3); + let mut decimal_builder = Decimal128Builder::with_capacity(20); for i in 110000..110010 { - decimal_builder.append_value(i as i128).unwrap(); + decimal_builder.append_value(i as i128); } for i in 100000..100010 { - decimal_builder.append_value(-i as i128).unwrap(); + decimal_builder.append_value(-i as i128); } - let array = decimal_builder.finish(); + let array = decimal_builder + .finish() + .with_precision_and_scale(10, 3) + .unwrap(); let schema = Schema::new(vec![Field::new("c1", array.data_type().clone(), true)]); RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap() } diff --git a/datafusion/core/tests/sql/mod.rs b/datafusion/core/tests/sql/mod.rs index e0167b5c4f3a..9d1c440a15df 100644 --- a/datafusion/core/tests/sql/mod.rs +++ b/datafusion/core/tests/sql/mod.rs @@ -560,9 +560,9 @@ async fn register_tpch_csv_data( DataType::Int64 => { cols.push(Box::new(Int64Builder::with_capacity(records.len()))) } - DataType::Decimal128(p, s) => cols.push(Box::new( - Decimal128Builder::with_capacity(records.len(), *p, *s), - )), + DataType::Decimal128(_, _) => { + cols.push(Box::new(Decimal128Builder::with_capacity(records.len()))) + } _ => { let msg = format!("Not implemented: {}", field.data_type()); Err(DataFusionError::Plan(msg))? @@ -600,7 +600,7 @@ async fn register_tpch_csv_data( .unwrap(); let val = val.trim().replace('.', ""); let value_i128 = val.parse::().unwrap(); - sb.append_value(value_i128)?; + sb.append_value(value_i128); } _ => Err(DataFusionError::Plan(format!( "Not implemented: {}", @@ -609,7 +609,21 @@ async fn register_tpch_csv_data( } } } - let cols: Vec = cols.iter_mut().map(|it| it.finish()).collect(); + let cols: Vec = cols + .iter_mut() + .zip(schema.fields()) + .map(|(it, field)| match field.data_type() { + DataType::Decimal128(p, s) => Arc::new( + it.as_any_mut() + .downcast_mut::() + .unwrap() + .finish() + .with_precision_and_scale(*p, *s) + .unwrap(), + ), + _ => it.finish(), + }) + .collect(); let batch = RecordBatch::try_new(Arc::clone(&schema), cols)?; @@ -879,14 +893,17 @@ pub fn table_with_decimal() -> Arc { } fn make_decimal() -> RecordBatch { - let mut decimal_builder = Decimal128Builder::with_capacity(20, 10, 3); + let mut decimal_builder = Decimal128Builder::with_capacity(20); for i in 110000..110010 { - decimal_builder.append_value(i as i128).unwrap(); + decimal_builder.append_value(i as i128); } for i in 100000..100010 { - decimal_builder.append_value(-i as i128).unwrap(); + decimal_builder.append_value(-i as i128); } - let array = decimal_builder.finish(); + let array = decimal_builder + .finish() + .with_precision_and_scale(10, 3) + .unwrap(); let schema = Schema::new(vec![Field::new("c1", array.data_type().clone(), true)]); RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap() } diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index 5d7350f720cd..d3b348c4bcb3 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -36,7 +36,7 @@ path = "src/lib.rs" [dependencies] ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } -arrow = { version = "24.0.0", default-features = false } +arrow = { version = "25.0.0", default-features = false } datafusion-common = { path = "../common", version = "13.0.0" } log = "^0.4" sqlparser = "0.25" diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml index 571ddb4bb600..1b4dc1059550 100644 --- a/datafusion/jit/Cargo.toml +++ b/datafusion/jit/Cargo.toml @@ -36,7 +36,7 @@ path = "src/lib.rs" jit = [] [dependencies] -arrow = { version = "24.0.0", default-features = false } +arrow = { version = "25.0.0", default-features = false } cranelift = "0.88.0" cranelift-jit = "0.88.0" cranelift-module = "0.88.0" diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml index ee4d4016dcd7..e07d798caf8d 100644 --- a/datafusion/optimizer/Cargo.toml +++ b/datafusion/optimizer/Cargo.toml @@ -37,7 +37,7 @@ default = ["unicode_expressions"] unicode_expressions = [] [dependencies] -arrow = { version = "24.0.0", features = ["prettyprint"] } +arrow = { version = "25.0.0", features = ["prettyprint"] } async-trait = "0.1.41" chrono = { version = "0.4", default-features = false } datafusion-common = { path = "../common", version = "13.0.0" } diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml index 4707b0e79125..ba4dc0b036a9 100644 --- a/datafusion/physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"] [dependencies] ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } -arrow = { version = "24.0.0", features = ["prettyprint"] } +arrow = { version = "25.0.0", features = ["prettyprint"] } blake2 = { version = "^0.10.2", optional = true } blake3 = { version = "1.0", optional = true } chrono = { version = "0.4", default-features = false } diff --git a/datafusion/physical-expr/src/aggregate/min_max.rs b/datafusion/physical-expr/src/aggregate/min_max.rs index 36d58c78008e..c415e7db5164 100644 --- a/datafusion/physical-expr/src/aggregate/min_max.rs +++ b/datafusion/physical-expr/src/aggregate/min_max.rs @@ -177,17 +177,17 @@ macro_rules! typed_min_max_batch_decimal128 { for i in 1..array.len() { result = result.$OP(array.value(i)); } - ScalarValue::Decimal128(Some(result.as_i128()), *$PRECISION, *$SCALE) + ScalarValue::Decimal128(Some(result), *$PRECISION, *$SCALE) } else { let mut result = 0_i128; let mut has_value = false; for i in 0..array.len() { if !has_value && array.is_valid(i) { has_value = true; - result = array.value(i).as_i128(); + result = array.value(i); } if array.is_valid(i) { - result = result.$OP(array.value(i).as_i128()); + result = result.$OP(array.value(i)); } } ScalarValue::Decimal128(Some(result), *$PRECISION, *$SCALE) diff --git a/datafusion/physical-expr/src/aggregate/sum.rs b/datafusion/physical-expr/src/aggregate/sum.rs index ca9b4c819eab..112856988129 100644 --- a/datafusion/physical-expr/src/aggregate/sum.rs +++ b/datafusion/physical-expr/src/aggregate/sum.rs @@ -169,7 +169,7 @@ fn sum_decimal_batch(values: &ArrayRef, precision: u8, scale: u8) -> Result s + v.as_i128(), + Some(v) => s + v, None => s, }); diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 665ef2d6828e..6db5b5a781d1 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -2152,19 +2152,14 @@ mod tests { precision: u8, scale: u8, ) -> Decimal128Array { - let mut decimal_builder = - Decimal128Builder::with_capacity(array.len(), precision, scale); - for value in array { - match value { - None => { - decimal_builder.append_null(); - } - Some(v) => { - decimal_builder.append_value(*v).expect("valid value"); - } - } + let mut decimal_builder = Decimal128Builder::with_capacity(array.len()); + for value in array.iter().copied() { + decimal_builder.append_option(value) } - decimal_builder.finish() + decimal_builder + .finish() + .with_precision_and_scale(precision, scale) + .unwrap() } #[test] diff --git a/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs b/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs index 99b8a5a06dd6..d73302d7e1fd 100644 --- a/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs +++ b/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs @@ -133,7 +133,7 @@ where { Ok(left .iter() - .map(|left| left.map(|left| op(left.as_i128(), right))) + .map(|left| left.map(|left| op(left, right))) .collect()) } @@ -152,7 +152,7 @@ where .zip(right.iter()) .map(|(left, right)| { if let (Some(left), Some(right)) = (left, right) { - Some(op(left.as_i128(), right.as_i128())) + Some(op(left, right)) } else { None } @@ -288,7 +288,7 @@ where .zip(right.iter()) .map(|(left, right)| { if let (Some(left), Some(right)) = (left, right) { - Some(op(left.as_i128(), right.as_i128())).transpose() + Some(op(left, right)).transpose() } else { Ok(None) } @@ -307,7 +307,7 @@ where left.iter() .map(|left| { if let Some(left) = left { - Some(op(left.as_i128(), right)).transpose() + Some(op(left, right)).transpose() } else { Ok(None) } @@ -444,19 +444,15 @@ mod tests { precision: u8, scale: u8, ) -> Decimal128Array { - let mut decimal_builder = - Decimal128Builder::with_capacity(array.len(), precision, scale); - for value in array { - match value { - None => { - decimal_builder.append_null(); - } - Some(v) => { - decimal_builder.append_value(*v).expect("valid value"); - } - } + let mut decimal_builder = Decimal128Builder::with_capacity(array.len()); + + for value in array.iter().copied() { + decimal_builder.append_option(value) } - decimal_builder.finish() + decimal_builder + .finish() + .with_precision_and_scale(precision, scale) + .unwrap() } #[test] diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index 5affc857c672..a1e87a38c2df 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -166,7 +166,6 @@ mod tests { TimestampNanosecondArray, UInt32Array, }, datatypes::*, - util::decimal::Decimal128, }; use datafusion_common::Result; @@ -286,20 +285,17 @@ mod tests { .collect::() .with_precision_and_scale(10, 3)?; - // closure that converts to i128 - let convert = |v: i128| Decimal128::new(20, 6, &v.to_le_bytes()); - generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal128(10, 3), Decimal128Array, DataType::Decimal128(20, 6), vec![ - Some(convert(1_234_000)), - Some(convert(2_222_000)), - Some(convert(3_000)), - Some(convert(4_000_000)), - Some(convert(5_000_000)), + Some(1_234_000), + Some(2_222_000), + Some(3_000), + Some(4_000_000), + Some(5_000_000), None, ], DEFAULT_DATAFUSION_CAST_OPTIONS @@ -310,20 +306,12 @@ mod tests { .collect::() .with_precision_and_scale(10, 3)?; - let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes()); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal128(10, 3), Decimal128Array, DataType::Decimal128(10, 2), - vec![ - Some(convert(123)), - Some(convert(222)), - Some(convert(0)), - Some(convert(400)), - Some(convert(500)), - None, - ], + vec![Some(123), Some(222), Some(0), Some(400), Some(500), None,], DEFAULT_DATAFUSION_CAST_OPTIONS ); @@ -476,115 +464,72 @@ mod tests { #[test] fn test_cast_numeric_to_decimal() -> Result<()> { // int8 - let convert = |v: i128| Decimal128::new(3, 0, &v.to_le_bytes()); generic_test_cast!( Int8Array, DataType::Int8, vec![1, 2, 3, 4, 5], Decimal128Array, DataType::Decimal128(3, 0), - vec![ - Some(convert(1)), - Some(convert(2)), - Some(convert(3)), - Some(convert(4)), - Some(convert(5)), - ], + vec![Some(1), Some(2), Some(3), Some(4), Some(5),], DEFAULT_DATAFUSION_CAST_OPTIONS ); // int16 - let convert = |v: i128| Decimal128::new(5, 0, &v.to_le_bytes()); generic_test_cast!( Int16Array, DataType::Int16, vec![1, 2, 3, 4, 5], Decimal128Array, DataType::Decimal128(5, 0), - vec![ - Some(convert(1)), - Some(convert(2)), - Some(convert(3)), - Some(convert(4)), - Some(convert(5)), - ], + vec![Some(1), Some(2), Some(3), Some(4), Some(5),], DEFAULT_DATAFUSION_CAST_OPTIONS ); // int32 - let convert = |v: i128| Decimal128::new(10, 0, &v.to_le_bytes()); generic_test_cast!( Int32Array, DataType::Int32, vec![1, 2, 3, 4, 5], Decimal128Array, DataType::Decimal128(10, 0), - vec![ - Some(convert(1)), - Some(convert(2)), - Some(convert(3)), - Some(convert(4)), - Some(convert(5)), - ], + vec![Some(1), Some(2), Some(3), Some(4), Some(5),], DEFAULT_DATAFUSION_CAST_OPTIONS ); // int64 - let convert = |v: i128| Decimal128::new(20, 0, &v.to_le_bytes()); generic_test_cast!( Int64Array, DataType::Int64, vec![1, 2, 3, 4, 5], Decimal128Array, DataType::Decimal128(20, 0), - vec![ - Some(convert(1)), - Some(convert(2)), - Some(convert(3)), - Some(convert(4)), - Some(convert(5)), - ], + vec![Some(1), Some(2), Some(3), Some(4), Some(5),], DEFAULT_DATAFUSION_CAST_OPTIONS ); // int64 to different scale - let convert = |v: i128| Decimal128::new(20, 2, &v.to_le_bytes()); generic_test_cast!( Int64Array, DataType::Int64, vec![1, 2, 3, 4, 5], Decimal128Array, DataType::Decimal128(20, 2), - vec![ - Some(convert(100)), - Some(convert(200)), - Some(convert(300)), - Some(convert(400)), - Some(convert(500)), - ], + vec![Some(100), Some(200), Some(300), Some(400), Some(500),], DEFAULT_DATAFUSION_CAST_OPTIONS ); // float32 - let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes()); generic_test_cast!( Float32Array, DataType::Float32, vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], Decimal128Array, DataType::Decimal128(10, 2), - vec![ - Some(convert(150)), - Some(convert(250)), - Some(convert(300)), - Some(convert(112)), - Some(convert(550)), - ], + vec![Some(150), Some(250), Some(300), Some(112), Some(550),], DEFAULT_DATAFUSION_CAST_OPTIONS ); // float64 - let convert = |v: i128| Decimal128::new(20, 4, &v.to_le_bytes()); generic_test_cast!( Float64Array, DataType::Float64, @@ -592,11 +537,11 @@ mod tests { Decimal128Array, DataType::Decimal128(20, 4), vec![ - Some(convert(15000)), - Some(convert(25000)), - Some(convert(30000)), - Some(convert(11234)), - Some(convert(55000)), + Some(15000), + Some(25000), + Some(30000), + Some(11234), + Some(55000), ], DEFAULT_DATAFUSION_CAST_OPTIONS ); diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs index 1e9fa7e9b343..2d34115220ec 100644 --- a/datafusion/physical-expr/src/expressions/in_list.rs +++ b/datafusion/physical-expr/src/expressions/in_list.rs @@ -238,7 +238,7 @@ macro_rules! collection_contains_check_decimal { if $CONTAINS_NULL { $ARRAY .iter() - .map(|vop| match vop.map(|v| !$VALUES.contains(&v.as_i128())) { + .map(|vop| match vop.map(|v| !$VALUES.contains(&v)) { Some(true) => None, x => x, }) @@ -246,7 +246,7 @@ macro_rules! collection_contains_check_decimal { } else { $ARRAY .iter() - .map(|vop| vop.map(|v| !$VALUES.contains(&v.as_i128()))) + .map(|vop| vop.map(|v| !$VALUES.contains(&v))) .collect::() } } else { @@ -254,7 +254,7 @@ macro_rules! collection_contains_check_decimal { if $CONTAINS_NULL { $ARRAY .iter() - .map(|vop| match vop.map(|v| $VALUES.contains(&v.as_i128())) { + .map(|vop| match vop.map(|v| $VALUES.contains(&v)) { Some(false) => None, x => x, }) @@ -262,7 +262,7 @@ macro_rules! collection_contains_check_decimal { } else { $ARRAY .iter() - .map(|vop| vop.map(|v| $VALUES.contains(&v.as_i128()))) + .map(|vop| vop.map(|v| $VALUES.contains(&v))) .collect::() } }; diff --git a/datafusion/physical-expr/src/expressions/try_cast.rs b/datafusion/physical-expr/src/expressions/try_cast.rs index 154ec85e58ec..92f345de72fd 100644 --- a/datafusion/physical-expr/src/expressions/try_cast.rs +++ b/datafusion/physical-expr/src/expressions/try_cast.rs @@ -121,7 +121,6 @@ mod tests { use arrow::array::{ Decimal128Array, Decimal128Builder, StringArray, Time64NanosecondArray, }; - use arrow::util::decimal::Decimal128; use arrow::{ array::{ Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, @@ -233,37 +232,28 @@ mod tests { // try cast one decimal data type to another decimal data type let array: Vec = vec![1234, 2222, 3, 4000, 5000]; let decimal_array = create_decimal_array(&array, 10, 3); - let convert = |v: i128| Decimal128::new(20, 6, &v.to_le_bytes()); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal128(10, 3), Decimal128Array, DataType::Decimal128(20, 6), vec![ - Some(convert(1_234_000)), - Some(convert(2_222_000)), - Some(convert(3_000)), - Some(convert(4_000_000)), - Some(convert(5_000_000)), + Some(1_234_000), + Some(2_222_000), + Some(3_000), + Some(4_000_000), + Some(5_000_000), None, ] ); let decimal_array = create_decimal_array(&array, 10, 3); - let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes()); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal128(10, 3), Decimal128Array, DataType::Decimal128(10, 2), - vec![ - Some(convert(123)), - Some(convert(222)), - Some(convert(0)), - Some(convert(400)), - Some(convert(500)), - None, - ] + vec![Some(123), Some(222), Some(0), Some(400), Some(500), None,] ); Ok(()) @@ -382,109 +372,66 @@ mod tests { #[test] fn test_try_cast_numeric_to_decimal() -> Result<()> { // int8 - let convert = |v: i128| Decimal128::new(3, 0, &v.to_le_bytes()); generic_test_cast!( Int8Array, DataType::Int8, vec![1, 2, 3, 4, 5], Decimal128Array, DataType::Decimal128(3, 0), - vec![ - Some(convert(1)), - Some(convert(2)), - Some(convert(3)), - Some(convert(4)), - Some(convert(5)), - ] + vec![Some(1), Some(2), Some(3), Some(4), Some(5),] ); // int16 - let convert = |v: i128| Decimal128::new(5, 0, &v.to_le_bytes()); generic_test_cast!( Int16Array, DataType::Int16, vec![1, 2, 3, 4, 5], Decimal128Array, DataType::Decimal128(5, 0), - vec![ - Some(convert(1)), - Some(convert(2)), - Some(convert(3)), - Some(convert(4)), - Some(convert(5)), - ] + vec![Some(1), Some(2), Some(3), Some(4), Some(5),] ); // int32 - let convert = |v: i128| Decimal128::new(10, 0, &v.to_le_bytes()); generic_test_cast!( Int32Array, DataType::Int32, vec![1, 2, 3, 4, 5], Decimal128Array, DataType::Decimal128(10, 0), - vec![ - Some(convert(1)), - Some(convert(2)), - Some(convert(3)), - Some(convert(4)), - Some(convert(5)), - ] + vec![Some(1), Some(2), Some(3), Some(4), Some(5),] ); // int64 - let convert = |v: i128| Decimal128::new(20, 0, &v.to_le_bytes()); generic_test_cast!( Int64Array, DataType::Int64, vec![1, 2, 3, 4, 5], Decimal128Array, DataType::Decimal128(20, 0), - vec![ - Some(convert(1)), - Some(convert(2)), - Some(convert(3)), - Some(convert(4)), - Some(convert(5)), - ] + vec![Some(1), Some(2), Some(3), Some(4), Some(5),] ); // int64 to different scale - let convert = |v: i128| Decimal128::new(20, 2, &v.to_le_bytes()); generic_test_cast!( Int64Array, DataType::Int64, vec![1, 2, 3, 4, 5], Decimal128Array, DataType::Decimal128(20, 2), - vec![ - Some(convert(100)), - Some(convert(200)), - Some(convert(300)), - Some(convert(400)), - Some(convert(500)), - ] + vec![Some(100), Some(200), Some(300), Some(400), Some(500),] ); // float32 - let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes()); generic_test_cast!( Float32Array, DataType::Float32, vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], Decimal128Array, DataType::Decimal128(10, 2), - vec![ - Some(convert(150)), - Some(convert(250)), - Some(convert(300)), - Some(convert(112)), - Some(convert(550)), - ] + vec![Some(150), Some(250), Some(300), Some(112), Some(550),] ); // float64 - let convert = |v: i128| Decimal128::new(20, 4, &v.to_le_bytes()); generic_test_cast!( Float64Array, DataType::Float64, @@ -492,11 +439,11 @@ mod tests { Decimal128Array, DataType::Decimal128(20, 4), vec![ - Some(convert(15000)), - Some(convert(25000)), - Some(convert(30000)), - Some(convert(11234)), - Some(convert(55000)), + Some(15000), + Some(25000), + Some(30000), + Some(11234), + Some(55000), ] ); Ok(()) @@ -577,12 +524,14 @@ mod tests { // create decimal array with the specified precision and scale fn create_decimal_array(array: &[i128], precision: u8, scale: u8) -> Decimal128Array { - let mut decimal_builder = - Decimal128Builder::with_capacity(array.len(), precision, scale); + let mut decimal_builder = Decimal128Builder::with_capacity(array.len()); for value in array { - decimal_builder.append_value(*value).expect("valid value"); + decimal_builder.append_value(*value); } decimal_builder.append_null(); - decimal_builder.finish() + decimal_builder + .finish() + .with_precision_and_scale(precision, scale) + .unwrap() } } diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index 569fe5c44936..ff2af33b22d8 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -40,7 +40,7 @@ default = [] json = ["pbjson", "pbjson-build", "serde", "serde_json"] [dependencies] -arrow = "24.0.0" +arrow = "25.0.0" datafusion = { path = "../core", version = "13.0.0" } datafusion-common = { path = "../common", version = "13.0.0" } datafusion-expr = { path = "../expr", version = "13.0.0" } diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml index bc32b07d9fb4..52e6e50c7e85 100644 --- a/datafusion/row/Cargo.toml +++ b/datafusion/row/Cargo.toml @@ -37,7 +37,7 @@ path = "src/lib.rs" jit = ["datafusion-jit"] [dependencies] -arrow = "24.0.0" +arrow = "25.0.0" datafusion-common = { path = "../common", version = "13.0.0" } datafusion-jit = { path = "../jit", version = "13.0.0", optional = true } paste = "^1.0" diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml index 4633e225ae51..4a88a01e31b0 100644 --- a/datafusion/sql/Cargo.toml +++ b/datafusion/sql/Cargo.toml @@ -37,7 +37,7 @@ default = ["unicode_expressions"] unicode_expressions = [] [dependencies] -arrow = { version = "24.0.0", default-features = false } +arrow = { version = "25.0.0", default-features = false } datafusion-common = { path = "../common", version = "13.0.0" } datafusion-expr = { path = "../expr", version = "13.0.0" } sqlparser = "0.25"