Implement is_identifier_(start/part) using icu_properties (#2865)

As mentioned in #2848 (comment), this uses our new default ICU4X data to replace `char::is_start` and `char::is_continue` from the `boa_unicode` crate with the [`icu_properties`](https://crates.io/crates/icu_properties) crate. Note that this doesn't deprecate `boa_unicode` yet, since that'll require some discussion about how to proceed with a now unused sub-crate.
boa-dev · Apr 24, 2023 · c330005 · c330005
1 parent cebec9d
commit c330005
Show file tree

Hide file tree

Showing 18 changed files with 97 additions and 197 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/boa_icu_provider/Cargo.toml b/boa_icu_provider/Cargo.toml
@@ -14,22 +14,26 @@ rust-version.workspace = true
 
 [dependencies]
 icu_provider = { version = "1.2.0", features = ["serde", "sync"] }
-icu_provider_blob = "1.2.0"
-icu_provider_adapters = { version = "1.2.0", features = ["serde"] }
 icu_collections = "1.2.0"
 icu_normalizer = "1.2.0"
+icu_properties = "1.2.0"
 zerovec = "0.9.4"
 
 icu_datagen = { version = "1.2.3", optional = true }
-once_cell = {version = "1.17.1", default-features = false, features = ["critical-section"], optional = true }
+icu_provider_blob = { version = "1.2.0", optional = true }
+icu_provider_adapters = { version = "1.2.0", features = ["serde"], optional = true }
+once_cell = { version = "1.17.1", default-features = false, features = [
+    "critical-section",
+], optional = true }
 log = { version = "0.4.17", optional = true }
 simple_logger = { version = "4.1.0", optional = true }
+icu_casemapping = { version = "0.7.2", optional = true }
 
 [features]
 default = ["std"]
-full = ["dep:once_cell"]
+full = ["dep:once_cell", "dep:icu_provider_blob", "dep:icu_provider_adapters"]
 std = ["once_cell?/std"]
-bin = ["dep:icu_datagen", "dep:simple_logger", "dep:log"]
+bin = ["dep:icu_datagen", "dep:simple_logger", "dep:log", "dep:icu_casemapping"]
 
 [[bin]]
 name = "boa-datagen"

diff --git a/boa_icu_provider/data/icudata.postcard b/boa_icu_provider/data/icudata.postcard
diff --git a/boa_icu_provider/data/min/fallback/likelysubtags_v1/und.rs.data b/boa_icu_provider/data/min/fallback/likelysubtags_v1/und.rs.data
diff --git a/boa_icu_provider/data/min/fallback/mod.rs b/boa_icu_provider/data/min/fallback/mod.rs
diff --git a/boa_icu_provider/data/min/fallback/parents_v1/und.rs.data b/boa_icu_provider/data/min/fallback/parents_v1/und.rs.data
diff --git a/boa_icu_provider/data/min/fallback/supplement/co_v1/mod.rs b/boa_icu_provider/data/min/fallback/supplement/co_v1/mod.rs
diff --git a/boa_icu_provider/data/min/fallback/supplement/co_v1/und.rs.data b/boa_icu_provider/data/min/fallback/supplement/co_v1/und.rs.data
diff --git a/boa_icu_provider/data/min/fallback/supplement/mod.rs b/boa_icu_provider/data/min/fallback/supplement/mod.rs
diff --git a/boa_icu_provider/data/min/mod.rs b/boa_icu_provider/data/min/mod.rs
@@ -1,9 +1,9 @@
 // @generated
 #[clippy::msrv = "1.61"]
-mod fallback;
-#[clippy::msrv = "1.61"]
 mod normalizer;
 #[clippy::msrv = "1.61"]
+mod props;
+#[clippy::msrv = "1.61"]
 use icu_provider::prelude::*;
 /// Implement [`DataProvider<M>`] on the given struct using the data
 /// hardcoded in this module. This allows the struct to be used with
@@ -96,63 +96,29 @@ macro_rules! impl_data_provider {
             }
         }
         #[clippy::msrv = "1.61"]
-        impl DataProvider<::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker> for $provider {
-            fn load(
-                &self,
-                req: DataRequest,
-            ) -> Result<DataResponse<::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker>, DataError> {
-                fallback::supplement::co_v1::lookup(&req.locale)
-                    .map(zerofrom::ZeroFrom::zero_from)
-                    .map(DataPayload::from_owned)
-                    .map(|payload| DataResponse {
-                        metadata: Default::default(),
-                        payload: Some(payload),
-                    })
-                    .ok_or_else(|| {
-                        DataErrorKind::MissingLocale.with_req(
-                            ::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker::KEY,
-                            req,
-                        )
-                    })
-            }
-        }
-        #[clippy::msrv = "1.61"]
-        impl DataProvider<::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker> for $provider {
-            fn load(
-                &self,
-                req: DataRequest,
-            ) -> Result<DataResponse<::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker>, DataError> {
-                fallback::likelysubtags_v1::lookup(&req.locale)
+        impl DataProvider<::icu_properties::provider::IdContinueV1Marker> for $provider {
+            fn load(&self, req: DataRequest) -> Result<DataResponse<::icu_properties::provider::IdContinueV1Marker>, DataError> {
+                props::idc_v1::lookup(&req.locale)
                     .map(zerofrom::ZeroFrom::zero_from)
                     .map(DataPayload::from_owned)
                     .map(|payload| DataResponse {
                         metadata: Default::default(),
                         payload: Some(payload),
                     })
-                    .ok_or_else(|| {
-                        DataErrorKind::MissingLocale.with_req(
-                            ::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker::KEY,
-                            req,
-                        )
-                    })
+                    .ok_or_else(|| DataErrorKind::MissingLocale.with_req(::icu_properties::provider::IdContinueV1Marker::KEY, req))
             }
         }
         #[clippy::msrv = "1.61"]
-        impl DataProvider<::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker> for $provider {
-            fn load(
-                &self,
-                req: DataRequest,
-            ) -> Result<DataResponse<::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker>, DataError> {
-                fallback::parents_v1::lookup(&req.locale)
+        impl DataProvider<::icu_properties::provider::IdStartV1Marker> for $provider {
+            fn load(&self, req: DataRequest) -> Result<DataResponse<::icu_properties::provider::IdStartV1Marker>, DataError> {
+                props::ids_v1::lookup(&req.locale)
                     .map(zerofrom::ZeroFrom::zero_from)
                     .map(DataPayload::from_owned)
                     .map(|payload| DataResponse {
                         metadata: Default::default(),
                         payload: Some(payload),
                     })
-                    .ok_or_else(|| {
-                        DataErrorKind::MissingLocale.with_req(::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker::KEY, req)
-                    })
+                    .ok_or_else(|| DataErrorKind::MissingLocale.with_req(::icu_properties::provider::IdStartV1Marker::KEY, req))
             }
         }
     };
@@ -185,21 +151,16 @@ macro_rules! impl_any_provider {
                     ::icu_normalizer::provider::CompatibilityDecompositionSupplementV1Marker::KEY.hashed();
                 const COMPATIBILITYDECOMPOSITIONTABLESV1MARKER: ::icu_provider::DataKeyHash =
                     ::icu_normalizer::provider::CompatibilityDecompositionTablesV1Marker::KEY.hashed();
-                const COLLATIONFALLBACKSUPPLEMENTV1MARKER: ::icu_provider::DataKeyHash =
-                    ::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker::KEY.hashed();
-                const LOCALEFALLBACKLIKELYSUBTAGSV1MARKER: ::icu_provider::DataKeyHash =
-                    ::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker::KEY.hashed();
-                const LOCALEFALLBACKPARENTSV1MARKER: ::icu_provider::DataKeyHash =
-                    ::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker::KEY.hashed();
+                const IDCONTINUEV1MARKER: ::icu_provider::DataKeyHash = ::icu_properties::provider::IdContinueV1Marker::KEY.hashed();
+                const IDSTARTV1MARKER: ::icu_provider::DataKeyHash = ::icu_properties::provider::IdStartV1Marker::KEY.hashed();
                 match key.hashed() {
                     CANONICALCOMPOSITIONSV1MARKER => normalizer::comp_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
                     CANONICALDECOMPOSITIONDATAV1MARKER => normalizer::nfd_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
                     CANONICALDECOMPOSITIONTABLESV1MARKER => normalizer::nfdex_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
                     COMPATIBILITYDECOMPOSITIONSUPPLEMENTV1MARKER => normalizer::nfkd_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
                     COMPATIBILITYDECOMPOSITIONTABLESV1MARKER => normalizer::nfkdex_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
-                    COLLATIONFALLBACKSUPPLEMENTV1MARKER => fallback::supplement::co_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
-                    LOCALEFALLBACKLIKELYSUBTAGSV1MARKER => fallback::likelysubtags_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
-                    LOCALEFALLBACKPARENTSV1MARKER => fallback::parents_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
+                    IDCONTINUEV1MARKER => props::idc_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
+                    IDSTARTV1MARKER => props::ids_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
                     _ => return Err(DataErrorKind::MissingDataKey.with_req(key, req)),
                 }
                 .map(|payload| AnyResponse {

diff --git a/...vider/data/min/fallback/parents_v1/mod.rs → ...icu_provider/data/min/props/idc_v1/mod.rs b/...vider/data/min/fallback/parents_v1/mod.rs → ...icu_provider/data/min/props/idc_v1/mod.rs
@@ -1,5 +1,6 @@
 // @generated
-type DataStruct = < :: icu_provider_adapters :: fallback :: provider :: LocaleFallbackParentsV1Marker as :: icu_provider :: DataMarker > :: Yokeable ;
+type DataStruct =
+    <::icu_properties::provider::IdContinueV1Marker as ::icu_provider::DataMarker>::Yokeable;
 pub fn lookup(locale: &icu_provider::DataLocale) -> Option<&'static DataStruct> {
     locale.is_empty().then(|| &UND)
 }

diff --git a/boa_icu_provider/data/min/props/idc_v1/und.rs.data b/boa_icu_provider/data/min/props/idc_v1/und.rs.data
diff --git a/...data/min/fallback/likelysubtags_v1/mod.rs → ...icu_provider/data/min/props/ids_v1/mod.rs b/...data/min/fallback/likelysubtags_v1/mod.rs → ...icu_provider/data/min/props/ids_v1/mod.rs
@@ -1,5 +1,6 @@
 // @generated
-type DataStruct = < :: icu_provider_adapters :: fallback :: provider :: LocaleFallbackLikelySubtagsV1Marker as :: icu_provider :: DataMarker > :: Yokeable ;
+type DataStruct =
+    <::icu_properties::provider::IdStartV1Marker as ::icu_provider::DataMarker>::Yokeable;
 pub fn lookup(locale: &icu_provider::DataLocale) -> Option<&'static DataStruct> {
     locale.is_empty().then(|| &UND)
 }