Merge remote-tracking branch 'origin/master'

arrow/Cargo.toml

@@ -51,7 +51,7 @@ chrono = "0.4"
 chrono-tz = {version = "0.4", optional = true}
 flatbuffers = { version = "=2.0.0", optional = true }
 hex = "0.4"
-comfy-table = { version = "4.0", optional = true, default-features = false }
+comfy-table = { version = "5.0", optional = true, default-features = false }
 pyo3 = { version = "0.14", optional = true }
 lexical-core = "^0.8"
 multiversion = "0.6.1"

arrow/src/array/builder.rs

@@ -1118,7 +1118,7 @@ pub struct FixedSizeBinaryBuilder {
     builder: FixedSizeListBuilder<UInt8Builder>,
 }
 
-const MAX_DECIMAL_FOR_EACH_PRECISION: [i128; 38] = [
+pub const MAX_DECIMAL_FOR_EACH_PRECISION: [i128; 38] = [
     9,
     99,
     999,
@@ -1158,7 +1158,7 @@ const MAX_DECIMAL_FOR_EACH_PRECISION: [i128; 38] = [
     9999999999999999999999999999999999999,
     170141183460469231731687303715884105727,
 ];
-const MIN_DECIMAL_FOR_EACH_PRECISION: [i128; 38] = [
+pub const MIN_DECIMAL_FOR_EACH_PRECISION: [i128; 38] = [
     -9,
     -99,
     -999,

arrow/src/array/mod.rs

@@ -393,6 +393,8 @@ pub use self::builder::StringBuilder;
 pub use self::builder::StringDictionaryBuilder;
 pub use self::builder::StructBuilder;
 pub use self::builder::UnionBuilder;
+pub use self::builder::MAX_DECIMAL_FOR_EACH_PRECISION;
+pub use self::builder::MIN_DECIMAL_FOR_EACH_PRECISION;
 
 pub type Int8Builder = PrimitiveBuilder<Int8Type>;
 pub type Int16Builder = PrimitiveBuilder<Int16Type>;

arrow/src/compute/kernels/sort.rs

@@ -1032,13 +1032,11 @@ fn sort_valids<T, U>(
 ) where
     T: ?Sized + Copy,
 {
-    let nulls_len = nulls.len();
+    let valids_len = valids.len();
     if !descending {
-        sort_unstable_by(valids, len.saturating_sub(nulls_len), |a, b| cmp(a.1, b.1));
+        sort_unstable_by(valids, len.min(valids_len), |a, b| cmp(a.1, b.1));
     } else {
-        sort_unstable_by(valids, len.saturating_sub(nulls_len), |a, b| {
-            cmp(a.1, b.1).reverse()
-        });
+        sort_unstable_by(valids, len.min(valids_len), |a, b| cmp(a.1, b.1).reverse());
         // reverse to keep a stable ordering
         nulls.reverse();
     }
@@ -1050,13 +1048,13 @@ fn sort_valids_array<T>(
     nulls: &mut [T],
     len: usize,
 ) {
-    let nulls_len = nulls.len();
+    let valids_len = valids.len();
     if !descending {
-        sort_unstable_by(valids, len.saturating_sub(nulls_len), |a, b| {
+        sort_unstable_by(valids, len.min(valids_len), |a, b| {
             cmp_array(a.1.as_ref(), b.1.as_ref())
         });
     } else {
-        sort_unstable_by(valids, len.saturating_sub(nulls_len), |a, b| {
+        sort_unstable_by(valids, len.min(valids_len), |a, b| {
             cmp_array(a.1.as_ref(), b.1.as_ref()).reverse()
         });
         // reverse to keep a stable ordering
@@ -1555,6 +1553,19 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_sort_to_indices_primitive_more_nulls_than_limit() {
+        test_sort_to_indices_primitive_arrays::<Int32Type>(
+            vec![None, None, Some(3), None, Some(1), None, Some(2)],
+            Some(SortOptions {
+                descending: false,
+                nulls_first: false,
+            }),
+            Some(2),
+            vec![4, 6],
+        );
+    }
+
     #[test]
     fn test_sort_boolean() {
         // boolean

arrow/src/compute/kernels/temporal.rs

@@ -95,6 +95,28 @@ pub fn using_chrono_tz(tz: &str) -> Option<FixedOffset> {
         .ok()
 }
 
+#[cfg(not(feature = "chrono-tz"))]
+pub fn using_chrono_tz_and_utc_naive_date_time(
+    _tz: &str,
+    _utc: chrono::NaiveDateTime,
+) -> Option<FixedOffset> {
+    Some(FixedOffset::east(0))
+}
+/// Parse the given string into a string representing fixed-offset that is correct as of the given
+/// UTC NaiveDateTime.
+/// Note that the offset is function of time and can vary depending on whether daylight savings is
+/// in effect or not. e.g. Australia/Sydney is +10:00 or +11:00 depending on DST.
+#[cfg(feature = "chrono-tz")]
+pub fn using_chrono_tz_and_utc_naive_date_time(
+    tz: &str,
+    utc: chrono::NaiveDateTime,
+) -> Option<FixedOffset> {
+    use chrono::{Offset, TimeZone};
+    tz.parse::<chrono_tz::Tz>()
+        .map(|tz| tz.offset_from_utc_datetime(&utc).fix())
+        .ok()
+}
+
 /// Extracts the hours of a given temporal array as an array of integers
 pub fn hour<T>(array: &PrimitiveArray<T>) -> Result<Int32Array>
 where
@@ -202,6 +224,8 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+    #[cfg(feature = "chrono-tz")]
+    use chrono::NaiveDate;
 
     #[test]
     fn test_temporal_array_date64_hour() {
@@ -435,4 +459,78 @@ mod tests {
         ));
         assert!(matches!(hour(&a), Err(ArrowError::ComputeError(_))))
     }
+
+    #[cfg(feature = "chrono-tz")]
+    #[test]
+    fn test_using_chrono_tz() {
+        let sydney_offset = FixedOffset::east(10 * 60 * 60);
+        assert_eq!(
+            using_chrono_tz(&"Australia/Sydney".to_string()),
+            Some(sydney_offset)
+        );
+
+        let nyc_offset = FixedOffset::west(5 * 60 * 60);
+        assert_eq!(
+            using_chrono_tz(&"America/New_York".to_string()),
+            Some(nyc_offset)
+        );
+    }
+
+    #[cfg(feature = "chrono-tz")]
+    #[test]
+    fn test_using_chrono_tz_and_utc_naive_date_time() {
+        let sydney_tz = "Australia/Sydney".to_string();
+        let sydney_offset_without_dst = FixedOffset::east(10 * 60 * 60);
+        let sydney_offset_with_dst = FixedOffset::east(11 * 60 * 60);
+        // Daylight savings ends
+        // When local daylight time was about to reach
+        // Sunday, 4 April 2021, 3:00:00 am clocks were turned backward 1 hour to
+        // Sunday, 4 April 2021, 2:00:00 am local standard time instead.
+
+        // Daylight savings starts
+        // When local standard time was about to reach
+        // Sunday, 3 October 2021, 2:00:00 am clocks were turned forward 1 hour to
+        // Sunday, 3 October 2021, 3:00:00 am local daylight time instead.
+
+        // Sydney 2021-04-04T02:30:00+11:00 is 2021-04-03T15:30:00Z
+        let utc_just_before_sydney_dst_ends =
+            NaiveDate::from_ymd(2021, 4, 3).and_hms_nano(15, 30, 0, 0);
+        assert_eq!(
+            using_chrono_tz_and_utc_naive_date_time(
+                &sydney_tz,
+                utc_just_before_sydney_dst_ends
+            ),
+            Some(sydney_offset_with_dst)
+        );
+        // Sydney 2021-04-04T02:30:00+10:00 is 2021-04-03T16:30:00Z
+        let utc_just_after_sydney_dst_ends =
+            NaiveDate::from_ymd(2021, 4, 3).and_hms_nano(16, 30, 0, 0);
+        assert_eq!(
+            using_chrono_tz_and_utc_naive_date_time(
+                &sydney_tz,
+                utc_just_after_sydney_dst_ends
+            ),
+            Some(sydney_offset_without_dst)
+        );
+        // Sydney 2021-10-03T01:30:00+10:00 is 2021-10-02T15:30:00Z
+        let utc_just_before_sydney_dst_starts =
+            NaiveDate::from_ymd(2021, 10, 2).and_hms_nano(15, 30, 0, 0);
+        assert_eq!(
+            using_chrono_tz_and_utc_naive_date_time(
+                &sydney_tz,
+                utc_just_before_sydney_dst_starts
+            ),
+            Some(sydney_offset_without_dst)
+        );
+        // Sydney 2021-04-04T03:30:00+11:00 is 2021-10-02T16:30:00Z
+        let utc_just_after_sydney_dst_starts =
+            NaiveDate::from_ymd(2022, 10, 2).and_hms_nano(16, 30, 0, 0);
+        assert_eq!(
+            using_chrono_tz_and_utc_naive_date_time(
+                &sydney_tz,
+                utc_just_after_sydney_dst_starts
+            ),
+            Some(sydney_offset_with_dst)
+        );
+    }
 }