diff --git a/CHANGELOG.md b/CHANGELOG.md index c8de9acb7ab..2e21bcc364b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## icu4x 1.4.x + - [Remove icu_datagen's dep on `fractional`](https://github.com/unicode-org/icu4x/pull/4472) + - `icu_datagen@1.4.1` + ## icu4x 1.4 (Nov 16, 2023) - General diff --git a/Cargo.lock b/Cargo.lock index 39789319364..5ffaa686392 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1090,15 +1090,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fraction" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59a78dd758a47a7305478e0e054f9fde4e983b9f9eccda162bf7ca03b79e9d40" -dependencies = [ - "num", -] - [[package]] name = "freertos-rust" version = "0.1.2" @@ -1547,7 +1538,7 @@ version = "1.4.0" [[package]] name = "icu_datagen" -version = "1.4.0" +version = "1.4.1" dependencies = [ "clap 4.2.1", "crlify", @@ -1556,7 +1547,6 @@ dependencies = [ "either", "elsa", "eyre", - "fraction", "icu", "icu_calendar", "icu_casemap", @@ -1588,6 +1578,7 @@ dependencies = [ "memchr", "ndarray", "num-bigint", + "num-rational", "once_cell", "postcard", "proc-macro2", @@ -2418,19 +2409,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "num" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" -dependencies = [ - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.4" @@ -2461,17 +2439,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-rational" version = "0.4.1" diff --git a/provider/datagen/Cargo.toml b/provider/datagen/Cargo.toml index b2ec669123a..8ea59fe24ec 100644 --- a/provider/datagen/Cargo.toml +++ b/provider/datagen/Cargo.toml @@ -26,7 +26,7 @@ edition.workspace = true homepage.workspace = true repository.workspace = true rust-version.workspace = true -version.workspace = true +version = "1.4.1" [package.metadata.docs.rs] all-features = true @@ -87,8 +87,6 @@ zip = { version = ">=0.5, <0.7", default-features = false, features = ["deflate" rayon = { version = "1.5", optional = true } ureq = { version = "2", optional = true } -fraction = {version = "0.14.0", default-features = false } -num-bigint = { version = "0.4.4", default-features = false } # Dependencies for "bin" feature @@ -102,6 +100,8 @@ crlify = { path = "../../utils/crlify" } icu = { path = "../../components/icu" } postcard = "1" simple_logger = { version = "4.1.0", default-features = false } +num-bigint = { version = "0.4.4", default-features = false } +num-rational = { version = "0.4", default-features = false } # Pre-experimental components with limited data generation icu_singlenumberformatter = { path = "../../experimental/singlenumberformatter", features = ["datagen"] } diff --git a/provider/datagen/src/transform/cldr/units/helpers.rs b/provider/datagen/src/transform/cldr/units/helpers.rs index 7d5f3a8cab9..839811729cb 100644 --- a/provider/datagen/src/transform/cldr/units/helpers.rs +++ b/provider/datagen/src/transform/cldr/units/helpers.rs @@ -6,10 +6,10 @@ use core::ops::{Div, Mul}; use core::str::FromStr; use std::collections::{BTreeMap, VecDeque}; -use fraction::GenericFraction; use icu_provider::DataError; use icu_unitsconversion::provider::{ConversionInfo, Exactness, Sign}; -use num_bigint::BigUint; +use num_bigint::BigInt; +use num_rational::Ratio; use crate::transform::cldr::cldr_serde::units::units_constants::Constant; @@ -273,9 +273,7 @@ pub fn process_constants<'a>( /// - " 1.5 E -2 " is converted to 15/1000 /// - " 1.5 E - 2" is an invalid scientific notation number /// - "1.5E-2.5" is an invalid scientific notation number -pub fn convert_scientific_notation_to_fraction( - number: &str, -) -> Result, DataError> { +pub fn convert_scientific_notation_to_fraction(number: &str) -> Result, DataError> { let mut parts = number.split('E'); let base = parts.next().unwrap_or("1").trim(); let exponent = parts.next().unwrap_or("0").trim(); @@ -285,23 +283,42 @@ pub fn convert_scientific_notation_to_fraction( )); } - let base = GenericFraction::::from_str(base) - .map_err(|_| DataError::custom("the base is not a valid decimal number"))?; + let mut split = base.split('.'); + let base_whole = split.next().ok_or(DataError::custom("the base is empty"))?; + let base_whole = if base_whole.is_empty() { + BigInt::from(0) + } else { + BigInt::from_str(base_whole).map_err(|_| { + DataError::custom("the whole part of the base is not a valid whole number") + })? + }; + + let base_dec = if let Some(dec_str) = split.next() { + let dec = BigInt::from_str(dec_str).map_err(|_| { + DataError::custom("the decimal part of the base is not a valid whole number") + })?; + let exp = dec_str.chars().filter(char::is_ascii_digit).count(); + Ratio::new(dec, BigInt::from(10u32).pow(exp as u32)) + } else { + Ratio::new(BigInt::from(0), BigInt::from(1)) + }; + + let base = base_dec + base_whole; + let exponent = i32::from_str(exponent) .map_err(|_| DataError::custom("the exponent is not a valid integer"))?; let result = if exponent >= 0 { - base.mul(GenericFraction::new( - BigUint::from(10u32).pow(exponent as u32), - BigUint::from(1u32), + base.mul(Ratio::new( + BigInt::from(10u32).pow(exponent as u32), + BigInt::from(1u32), )) } else { - base.div(GenericFraction::new( - BigUint::from(10u32).pow((-exponent) as u32), - BigUint::from(1u32), + base.div(Ratio::new( + BigInt::from(10u32).pow((-exponent) as u32), + BigInt::from(1u32), )) }; - Ok(result) } @@ -309,27 +326,27 @@ pub fn convert_scientific_notation_to_fraction( #[test] fn test_convert_scientific_notation_to_fraction() { let input = "1E2"; - let expected = GenericFraction::::new(BigUint::from(100u32), BigUint::from(1u32)); + let expected = Ratio::::new(BigInt::from(100u32), BigInt::from(1u32)); let actual = convert_scientific_notation_to_fraction(input).unwrap(); assert_eq!(expected, actual); let input = "1E-2"; - let expected = GenericFraction::::new(BigUint::from(1u32), BigUint::from(100u32)); + let expected = Ratio::::new(BigInt::from(1u32), BigInt::from(100u32)); let actual = convert_scientific_notation_to_fraction(input).unwrap(); assert_eq!(expected, actual); let input = "1.5E2"; - let expected = GenericFraction::::new(BigUint::from(150u32), BigUint::from(1u32)); + let expected = Ratio::::new(BigInt::from(150u32), BigInt::from(1u32)); let actual = convert_scientific_notation_to_fraction(input).unwrap(); assert_eq!(expected, actual); let input = "1.5E-2"; - let expected = GenericFraction::::new(BigUint::from(15u32), BigUint::from(1000u32)); + let expected = Ratio::::new(BigInt::from(15u32), BigInt::from(1000u32)); let actual = convert_scientific_notation_to_fraction(input).unwrap(); assert_eq!(expected, actual); let input = " 1.5 E -2 "; - let expected = GenericFraction::::new(BigUint::from(15u32), BigUint::from(1000u32)); + let expected = Ratio::::new(BigInt::from(15u32), BigInt::from(1000u32)); let actual = convert_scientific_notation_to_fraction(input).unwrap(); assert_eq!(expected, actual); @@ -340,6 +357,11 @@ fn test_convert_scientific_notation_to_fraction() { let input = "1.5E-2.5"; let actual = convert_scientific_notation_to_fraction(input); assert!(actual.is_err()); + + let input = "0.308"; + let expected = Ratio::::new(BigInt::from(308), BigInt::from(1000u32)); + let actual = convert_scientific_notation_to_fraction(input).unwrap(); + assert_eq!(expected, actual); } /// Determines if a string contains any alphabetic characters. @@ -390,28 +412,19 @@ pub fn is_scientific_number(s: &str) -> bool { } /// Transforms a fractional number into byte numerators, byte denominators, and a sign. -pub fn flatten_fraction( - fraction: GenericFraction, -) -> Result<(Vec, Vec, Sign), DataError> { - let numerator = match fraction.numer() { - Some(numerator) => numerator.to_bytes_le(), - None => return Err(DataError::custom("the numerator is too large")), - }; - - let denominator = match fraction.denom() { - Some(denominator) => denominator.to_bytes_le(), - None => return Err(DataError::custom("the denominator is too large")), - }; - - let sign = match fraction.sign() { - Some(fraction::Sign::Plus) => Sign::Positive, - Some(fraction::Sign::Minus) => Sign::Negative, - None => { - return Err(DataError::custom("the sign is not defined")); - } +pub fn flatten_fraction(fraction: Ratio) -> Result<(Vec, Vec, Sign), DataError> { + let (n_sign, numer) = fraction.numer().to_bytes_le(); + let (d_sign, denom) = fraction.denom().to_bytes_le(); + + // Ratio's constructor sets denom > 0 but it's worth + // checking in case we decide to switch to new_raw() to avoid reducing + let sign = if n_sign * d_sign == num_bigint::Sign::Minus { + Sign::Negative + } else { + Sign::Positive }; - Ok((numerator, denominator, sign)) + Ok((numer, denom, sign)) } /// Converts slices of numerator and denominator strings to a fraction. @@ -425,8 +438,8 @@ pub fn flatten_fraction( pub fn convert_slices_to_fraction( numerator_strings: &[&str], denominator_strings: &[&str], -) -> Result, DataError> { - let mut fraction = GenericFraction::new(BigUint::from(1u32), BigUint::from(1u32)); +) -> Result, DataError> { + let mut fraction = Ratio::new(BigInt::from(1u32), BigInt::from(1u32)); for numerator in numerator_strings { let num_fraction = convert_scientific_notation_to_fraction(numerator)?; @@ -446,19 +459,19 @@ pub fn convert_slices_to_fraction( fn test_convert_array_of_strings_to_fraction() { let numerator: Vec<&str> = vec!["1"]; let denominator: Vec<&str> = vec!["2"]; - let expected = GenericFraction::new(BigUint::from(1u32), BigUint::from(2u32)); + let expected = Ratio::new(BigInt::from(1i32), BigInt::from(2i32)); let actual = convert_slices_to_fraction(&numerator, &denominator).unwrap(); assert_eq!(expected, actual); let numerator = vec!["1", "2"]; let denominator = vec!["3", "1E2"]; - let expected = GenericFraction::new(BigUint::from(2u32), BigUint::from(300u32)); + let expected = Ratio::new(BigInt::from(2i32), BigInt::from(300i32)); let actual = convert_slices_to_fraction(&numerator, &denominator).unwrap(); assert_eq!(expected, actual); let numerator = vec!["1", "2"]; let denominator = vec!["3", "1E-2"]; - let expected = GenericFraction::new(BigUint::from(200u32), BigUint::from(3u32)); + let expected = Ratio::new(BigInt::from(200i32), BigInt::from(3i32)); let actual = convert_slices_to_fraction(&numerator, &denominator).unwrap(); assert_eq!(expected, actual); @@ -469,13 +482,13 @@ fn test_convert_array_of_strings_to_fraction() { let numerator = vec!["1E2"]; let denominator = vec!["2"]; - let expected = GenericFraction::new(BigUint::from(50u32), BigUint::from(1u32)); + let expected = Ratio::new(BigInt::from(50i32), BigInt::from(1i32)); let actual = convert_slices_to_fraction(&numerator, &denominator).unwrap(); assert_eq!(expected, actual); let numerator = vec!["1E2", "2"]; let denominator = vec!["3", "1E2"]; - let expected = GenericFraction::new(BigUint::from(2u32), BigUint::from(3u32)); + let expected = Ratio::new(BigInt::from(2i32), BigInt::from(3i32)); let actual = convert_slices_to_fraction(&numerator, &denominator).unwrap(); assert_eq!(expected, actual); } diff --git a/provider/datagen/src/transform/cldr/units/info.rs b/provider/datagen/src/transform/cldr/units/info.rs index bab66456177..ba527bc26e3 100644 --- a/provider/datagen/src/transform/cldr/units/info.rs +++ b/provider/datagen/src/transform/cldr/units/info.rs @@ -5,7 +5,6 @@ use std::{borrow::Cow, collections::BTreeMap}; use crate::transform::cldr::cldr_serde::{self}; -use fraction::{GenericFraction, Zero}; use icu_provider::{ datagen::IterableDataProvider, DataError, DataLocale, DataPayload, DataProvider, DataRequest, DataResponse, @@ -116,6 +115,7 @@ fn test_basic() { use icu_provider::prelude::*; use icu_unitsconversion::provider::*; use num_bigint::BigUint; + use num_rational::Ratio; use zerofrom::ZeroFrom; use zerovec::maps::ZeroVecLike; @@ -157,7 +157,7 @@ fn test_basic() { factor_num: ZeroVec::from(big_one.to_bytes_le()), factor_den: ZeroVec::from(big_one.to_bytes_le()), offset_sign: Sign::Positive, - offset_num: ZeroVec::from(BigUint::zero().to_bytes_le()), + offset_num: ZeroVec::from(BigUint::from(0u32).to_bytes_le()), offset_den: ZeroVec::from(big_one.to_bytes_le()), exactness: Exactness::Exact, } @@ -167,17 +167,17 @@ fn test_basic() { let foot_convert_index = foot.convert_info.get().unwrap().as_unsigned_int() as usize; let foot_convert_ule = convert_units.zvl_get(foot_convert_index).unwrap(); let foot_convert: ConversionInfo = ZeroFrom::zero_from(foot_convert_ule); - let ft_to_m = GenericFraction::::new(BigUint::from(3048u32), BigUint::from(10000u32)); + let ft_to_m = Ratio::new(BigUint::from(3048u32), BigUint::from(10000u32)); assert_eq!( foot_convert, ConversionInfo { base_unit: Cow::Borrowed("meter"), factor_sign: Sign::Positive, - factor_num: ZeroVec::from(ft_to_m.numer().unwrap().to_bytes_le()), - factor_den: ZeroVec::from(ft_to_m.denom().unwrap().to_bytes_le()), + factor_num: ZeroVec::from(ft_to_m.numer().to_bytes_le()), + factor_den: ZeroVec::from(ft_to_m.denom().to_bytes_le()), offset_sign: Sign::Positive, - offset_num: ZeroVec::from(BigUint::zero().to_bytes_le()), + offset_num: ZeroVec::from(BigUint::from(0u32).to_bytes_le()), offset_den: ZeroVec::from(big_one.to_bytes_le()), exactness: Exactness::Exact, }