Replace SerdeDeDataProvider with BufferProvider (#1369)

- Add buffer_format to DataResponseMetadata
- Migrate fs provider to BufferFormat enum
- impl BufferProvider for FsDataProvider
- Migrate FsDataProvider to use DeserializingProvider for deserialization
- Rename serialization primitives
- Rename features

experimental/segmenter/Cargo.toml

@@ -36,7 +36,7 @@ criterion = "0.3"
 serde-json-core = { version = "0.4", features = ["std"] }
 
 [build-dependencies]
-icu_provider_fs = { version = "0.4", path = "../../provider/fs", features = ["provider_json"] }
+icu_provider_fs = { version = "0.4", path = "../../provider/fs", features = ["deserialize_json"] }
 icu = { version = "0.4", path = "../../components/icu" }
 icu_testdata = { version = "0.4", path = "../../provider/testdata" }
 serde = { version = "1.0", features = ["derive"] }

ffi/diplomat/Cargo.toml

@@ -43,7 +43,7 @@ path = "src/lib.rs"
 default = ["provider_fs", "provider_static"]
 wearos = ["smaller_static", "freertos"]
 
-provider_fs = ["icu_provider_fs", "icu_provider_fs/provider_json"]
+provider_fs = ["icu_provider_fs", "icu_provider_fs/deserialize_json"]
 provider_static = ["icu_testdata"]
 smaller_static = ["provider_static"]
 
@@ -65,7 +65,7 @@ icu_locid = { path = "../../components/locid" }
 icu_locid_macros = { path = "../../components/locid/macros" }
 icu_plurals = { path = "../../components/plurals/" }
 icu_properties = { path = "../../components/properties/" }
-icu_provider = { path = "../../provider/core", features = ["provider_serde"] }
+icu_provider = { path = "../../provider/core", features = ["serde"] }
 icu_provider_blob = { path = "../../provider/blob" }
 tinystr = { version = "0.4.10", features = ["alloc"], default-features = false }
 writeable = { path = "../../utils/writeable/" }

ffi/diplomat/src/decimal.rs

@@ -67,8 +67,9 @@ pub mod ffi {
             provider: &ICU4XDataProvider,
             options: ICU4XFixedDecimalFormatOptions,
         ) -> ICU4XFixedDecimalFormatResult {
-            let provider = provider.0.as_ref();
-            Self::try_new_impl(locale, provider, options)
+            use icu_provider::serde::AsDeserializingBufferProvider;
+            let provider = provider.0.as_deserializing();
+            Self::try_new_impl(locale, &provider, options)
         }
 
         /// Creates a new [`ICU4XFixedDecimalFormat`] from a [`ICU4XStaticDataProvider`].

ffi/diplomat/src/locale_canonicalizer.rs

@@ -35,8 +35,9 @@ pub mod ffi {
         /// Create a new [`ICU4XLocaleCanonicalizer`].
         /// See [the Rust docs](https://unicode-org.github.io/icu4x-docs/doc/icu/locale_canonicalizer/struct.LocaleCanonicalizer.html#method.new) for more details.
         pub fn create(provider: &ICU4XDataProvider) -> Option<Box<ICU4XLocaleCanonicalizer>> {
-            let provider = provider.0.as_ref();
-            LocaleCanonicalizer::new(provider)
+            use icu_provider::serde::AsDeserializingBufferProvider;
+            let provider = provider.0.as_deserializing();
+            LocaleCanonicalizer::new(&provider)
                 .ok()
                 .map(|lc| Box::new(ICU4XLocaleCanonicalizer(lc)))
         }

ffi/diplomat/src/pluralrules.rs

@@ -54,8 +54,9 @@ pub mod ffi {
             provider: &ICU4XDataProvider,
             ty: ICU4XPluralRuleType,
         ) -> ICU4XCreatePluralRulesResult {
-            let provider = provider.0.as_ref();
-            Self::try_new_impl(locale, provider, ty)
+            use icu_provider::serde::AsDeserializingBufferProvider;
+            let provider = provider.0.as_deserializing();
+            Self::try_new_impl(locale, &provider, ty)
         }
 
         /// Creates a new [`ICU4XPluralRules`] from a [`ICU4XStaticDataProvider`].

ffi/diplomat/src/properties_maps.rs

@@ -30,8 +30,9 @@ pub mod ffi {
         /// Gets a map for Unicode property Script from a [`ICU4XDataProvider`].
         /// See [the Rust docs](https://unicode-org.github.io/icu4x-docs/doc/icu_properties/maps/fn.get_script.html) for more information.
         pub fn try_get_script(provider: &ICU4XDataProvider) -> ICU4XCodePointMapData16Response {
-            let provider = provider.0.as_ref();
-            Self::prepare_result_from_script(maps::get_script(provider))
+            use icu_provider::serde::AsDeserializingBufferProvider;
+            let provider = provider.0.as_deserializing();
+            Self::prepare_result_from_script(maps::get_script(&provider))
         }
 
         /// Gets a map for Unicode property Script from a [`ICU4XStaticDataProvider`].

ffi/diplomat/src/properties_sets.rs

@@ -31,8 +31,9 @@ pub mod ffi {
         pub fn try_get_ascii_hex_digit(
             provider: &ICU4XDataProvider,
         ) -> ICU4XCodePointSetDataResult {
-            let provider = provider.0.as_ref();
-            Self::prepare_result(sets::get_ascii_hex_digit(provider))
+            use icu_provider::serde::AsDeserializingBufferProvider;
+            let provider = provider.0.as_deserializing();
+            Self::prepare_result(sets::get_ascii_hex_digit(&provider))
         }
 
         /// Gets a set for Unicode property ascii_hex_digit from a [`ICU4XStaticDataProvider`].

ffi/diplomat/src/provider.rs

@@ -11,7 +11,7 @@ pub mod ffi {
     ))]
     use alloc::string::ToString;
 
-    use icu_provider::serde::SerdeDeDataProvider;
+    use icu_provider::prelude::BufferProvider;
     use icu_provider_blob::BlobDataProvider;
     use icu_provider_blob::StaticDataProvider;
     #[cfg(all(
@@ -23,7 +23,7 @@ pub mod ffi {
     #[diplomat::opaque]
     /// An ICU4X data provider, capable of loading ICU4X data keys from some source.
     /// See [the Rust docs](https://unicode-org.github.io/icu4x-docs/doc/icu_provider/prelude/trait.DataProvider.html) for more information.
-    pub struct ICU4XDataProvider(pub Box<dyn SerdeDeDataProvider + 'static>);
+    pub struct ICU4XDataProvider(pub Box<dyn BufferProvider + 'static>);
 
     /// A result type for `ICU4XDataProvider::create`.
     pub struct ICU4XCreateDataProviderResult {
@@ -43,13 +43,10 @@ pub mod ffi {
                 not(any(target_arch = "wasm32", target_os = "none"))
             ))]
             match FsDataProvider::try_new(path.to_string()) {
-                Ok(fs) => {
-                    let erased = Box::new(fs);
-                    ICU4XCreateDataProviderResult {
-                        provider: Some(Box::new(ICU4XDataProvider(erased))),
-                        success: true,
-                    }
-                }
+                Ok(fs) => ICU4XCreateDataProviderResult {
+                    provider: Some(Box::new(ICU4XDataProvider(Box::new(fs)))),
+                    success: true,
+                },
                 Err(_) => ICU4XCreateDataProviderResult {
                     provider: None,
                     success: false,
@@ -75,9 +72,8 @@ pub mod ffi {
                 let provider = icu_testdata::get_smaller_static_provider();
                 #[cfg(not(feature = "smaller_static"))]
                 let provider = icu_testdata::get_static_provider();
-                let erased = Box::new(provider);
                 ICU4XCreateDataProviderResult {
-                    provider: Some(Box::new(ICU4XDataProvider(erased))),
+                    provider: Some(Box::new(ICU4XDataProvider(Box::new(provider)))),
                     success: true,
                 }
             }

ffi/diplomat/wasm/test/data-providers.mjs

@@ -15,12 +15,14 @@ test("use create_from_byte_slice to format a simple decimal", async t => {
   const bytes = new Uint8Array(nodeBuffer.buffer, nodeBuffer.byteOffset, nodeBuffer.length);
   const result = ICU4XDataProvider.create_from_byte_slice(bytes);
   t.assert(result.success);
-  const format = ICU4XFixedDecimalFormat.try_new(locale, result.provider, ICU4XFixedDecimalFormatOptions.default()).fdf;
+
+  const format = ICU4XFixedDecimalFormat.try_new(locale, result.provider, ICU4XFixedDecimalFormatOptions.default());
+  t.assert(format.success);
 
   const decimal = ICU4XFixedDecimal.create(1234);
   decimal.multiply_pow10(-2);
 
-  t.is(format.format(decimal), "১২.৩৪");
+  t.is(format.fdf.format(decimal), "১২.৩৪");
 });
 
 test("fail to create from invalid buffer", t => {

provider/blob/Cargo.toml

@@ -27,7 +27,7 @@ include = [
 all-features = true
 
 [dependencies]
-icu_provider = { version = "0.4", path = "../../provider/core", features = ["provider_serde"] }
+icu_provider = { version = "0.4", path = "../../provider/core", features = ["deserialize_postcard_07"] }
 icu_locid = { version = "0.4", path = "../../components/locid", features = ["serde"] }
 serde = { version = "1.0", default-features = false, features = ["alloc"] }
 postcard = { version = "0.7.0", default-features = false }
@@ -49,5 +49,5 @@ icu_locid_macros = { version = "0.4", path = "../../components/locid/macros" }
 path = "src/lib.rs"
 
 [features]
-export = ["log", "postcard/alloc", "std", "litemap"]
+export = ["log", "postcard/alloc", "std", "litemap", "icu_provider/serialize"]
 std = ["icu_locid/std", "icu_provider/std"]

provider/blob/src/blob_data_provider.rs

@@ -6,8 +6,8 @@ use crate::blob_schema::BlobSchema;
 use crate::path_util;
 use alloc::rc::Rc;
 use alloc::string::String;
+use icu_provider::buffer_provider::BufferFormat;
 use icu_provider::prelude::*;
-use icu_provider::serde::{SerdeDeDataProvider, SerdeDeDataReceiver};
 use serde::de::Deserialize;
 use yoke::trait_hack::YokeTraitHack;
 use yoke::*;
@@ -111,28 +111,25 @@ where
                     .map_err(DataError::new_resc_error)?;
                 Ok(data.0)
             })?;
+        let mut metadata = DataResponseMetadata::default();
+        // TODO(#1109): Set metadata.data_langid correctly.
+        metadata.buffer_format = Some(BufferFormat::Postcard07);
         Ok(DataResponse {
-            metadata: DataResponseMetadata {
-                data_langid: req.resource_path.options.langid.clone(),
-            },
+            metadata,
             payload: Some(payload),
         })
     }
 }
 
-impl SerdeDeDataProvider for BlobDataProvider {
-    fn load_to_receiver(
-        &self,
-        req: &DataRequest,
-        receiver: &mut dyn SerdeDeDataReceiver,
-    ) -> Result<DataResponseMetadata, DataError> {
-        let file = self.get_file(req)?;
-        receiver.receive_yoked_buffer(file, |bytes, f2| {
-            let mut d = postcard::Deserializer::from_bytes(bytes);
-            f2(&mut <dyn erased_serde::Deserializer>::erase(&mut d))
-        })?;
-        Ok(DataResponseMetadata {
-            data_langid: req.resource_path.options.langid.clone(),
+impl BufferProvider for BlobDataProvider {
+    fn load_buffer(&self, req: DataRequest) -> Result<DataResponse<BufferMarker>, DataError> {
+        let yoked_buffer = self.get_file(&req)?;
+        let mut metadata = DataResponseMetadata::default();
+        // TODO(#1109): Set metadata.data_langid correctly.
+        metadata.buffer_format = Some(BufferFormat::Postcard07);
+        Ok(DataResponse {
+            metadata,
+            payload: Some(DataPayload::from_yoked_buffer(yoked_buffer)),
         })
     }
 }

provider/blob/src/export/blob_exporter.rs

@@ -6,7 +6,7 @@ use crate::blob_schema::*;
 use crate::path_util;
 use icu_provider::export::DataExporter;
 use icu_provider::prelude::*;
-use icu_provider::serde::SerdeSeDataStructMarker;
+use icu_provider::serde::SerializeMarker;
 use litemap::LiteMap;
 use zerovec::ZeroMap;
 
@@ -35,25 +35,19 @@ impl Drop for BlobExporter<'_> {
     }
 }
 
-/// TODO(#837): De-duplicate this code from icu_provider_fs.
-fn serialize(obj: &dyn erased_serde::Serialize) -> Result<Vec<u8>, DataError> {
-    let mut serializer = postcard::Serializer {
-        output: postcard::flavors::AllocVec(Vec::new()),
-    };
-    obj.erased_serialize(&mut <dyn erased_serde::Serializer>::erase(&mut serializer))?;
-    Ok(serializer.output.0)
-}
-
-impl DataExporter<SerdeSeDataStructMarker> for BlobExporter<'_> {
+impl DataExporter<SerializeMarker> for BlobExporter<'_> {
     fn put_payload(
         &mut self,
         req: DataRequest,
-        obj: DataPayload<SerdeSeDataStructMarker>,
+        payload: DataPayload<SerializeMarker>,
     ) -> Result<(), DataError> {
         let path = path_util::resource_path_to_string(&req.resource_path);
         log::trace!("Adding: {}", path);
-        let buffer = serialize(obj.get().as_serialize())?;
-        self.resources.insert(path, buffer);
+        let mut serializer = postcard::Serializer {
+            output: postcard::flavors::AllocVec(Vec::new()),
+        };
+        payload.serialize(&mut <dyn erased_serde::Serializer>::erase(&mut serializer))?;
+        self.resources.insert(path, serializer.output.0);
         Ok(())
     }
 
@@ -67,8 +61,13 @@ impl DataExporter<SerdeSeDataStructMarker> for BlobExporter<'_> {
             resources: zm.as_borrowed(),
         });
         log::info!("Serializing blob to output stream...");
-        let vec = serialize(&blob)?;
-        self.sink.write(&vec).map_err(|e| e.to_string())?;
+        let mut serializer = postcard::Serializer {
+            output: postcard::flavors::AllocVec(Vec::new()),
+        };
+        serde::Serialize::serialize(&blob, &mut serializer).map_err(DataError::new_resc_error)?;
+        self.sink
+            .write(&serializer.output.0)
+            .map_err(|e| e.to_string())?;
         self.resources.clear();
         Ok(())
     }

provider/blob/src/static_data_provider.rs

@@ -4,8 +4,8 @@
 
 use crate::blob_schema::*;
 use crate::path_util;
+use icu_provider::buffer_provider::BufferFormat;
 use icu_provider::prelude::*;
-use icu_provider::serde::{SerdeDeDataProvider, SerdeDeDataReceiver};
 use serde::de::Deserialize;
 use zerovec::map::ZeroMapBorrowed;
 
@@ -117,28 +117,25 @@ where
         let file = self.get_file(req)?;
         let data = M::Yokeable::deserialize(&mut postcard::Deserializer::from_bytes(file))
             .map_err(DataError::new_resc_error)?;
+        let mut metadata = DataResponseMetadata::default();
+        // TODO(#1109): Set metadata.data_langid correctly.
+        metadata.buffer_format = Some(BufferFormat::Postcard07);
         Ok(DataResponse {
-            metadata: DataResponseMetadata {
-                data_langid: req.resource_path.options.langid.clone(),
-            },
+            metadata,
             payload: Some(DataPayload::from_owned(data)),
         })
     }
 }
 
-impl SerdeDeDataProvider for StaticDataProvider {
-    fn load_to_receiver(
-        &self,
-        req: &DataRequest,
-        receiver: &mut dyn SerdeDeDataReceiver,
-    ) -> Result<DataResponseMetadata, DataError> {
-        let file = self.get_file(req)?;
-        receiver.receive_static(&mut <dyn erased_serde::Deserializer>::erase(
-            &mut postcard::Deserializer::from_bytes(file),
-        ))?;
-
-        Ok(DataResponseMetadata {
-            data_langid: req.resource_path.options.langid.clone(),
+impl BufferProvider for StaticDataProvider {
+    fn load_buffer(&self, req: DataRequest) -> Result<DataResponse<BufferMarker>, DataError> {
+        let static_buffer = self.get_file(&req)?;
+        let mut metadata = DataResponseMetadata::default();
+        // TODO(#1109): Set metadata.data_langid correctly.
+        metadata.buffer_format = Some(BufferFormat::Postcard07);
+        Ok(DataResponse {
+            metadata,
+            payload: Some(DataPayload::from_static_buffer(static_buffer)),
         })
     }
 }

provider/cldr/Cargo.toml

@@ -36,7 +36,7 @@ extra_features = [
 all-features = true
 
 [dependencies]
-icu_provider = { version = "0.4", path = "../../provider/core", features = ["provider_serde"] }
+icu_provider = { version = "0.4", path = "../../provider/core", features = ["serialize"] }
 icu_locid = { version = "0.4", path = "../../components/locid" }
 icu_plurals = { version = "0.4", path = "../../components/plurals" }
 icu_datetime = { version = "0.4", path = "../../components/datetime", features = ["provider_transform_internals"] }

provider/cldr/src/support.rs

@@ -5,7 +5,7 @@
 use crate::CldrPaths;
 use icu_provider::iter::{IterableDataProviderCore, KeyedDataProvider};
 use icu_provider::prelude::*;
-use icu_provider::serde::SerdeSeDataStructMarker;
+use icu_provider::serde::SerializeMarker;
 use std::convert::TryFrom;
 use std::sync::RwLock;
 
@@ -34,7 +34,7 @@ fn map_poison<E>(_err: E) -> DataError {
 /// A lazy-initialized CLDR JSON data provider.
 impl<'b, T> LazyCldrProvider<T>
 where
-    T: DataProvider<SerdeSeDataStructMarker>
+    T: DataProvider<SerializeMarker>
         + IterableDataProviderCore
         + KeyedDataProvider
         + TryFrom<&'b dyn CldrPaths>,
@@ -45,7 +45,7 @@ where
         &self,
         req: &DataRequest,
         cldr_paths: &'b dyn CldrPaths,
-    ) -> Result<Option<DataResponse<SerdeSeDataStructMarker>>, DataError> {
+    ) -> Result<Option<DataResponse<SerializeMarker>>, DataError> {
         if T::supports_key(&req.resource_path.key).is_err() {
             return Ok(None);
         }

provider/cldr/src/transform/datetime/patterns.rs

@@ -44,10 +44,10 @@ impl DataProvider<calendar::DatePatternsV1Marker> for DatePatternsProvider {
     ) -> Result<DataResponse<calendar::DatePatternsV1Marker>, DataError> {
         DatePatternsProvider::supports_key(&req.resource_path.key)?;
         let dates = self.0.dates_for(req)?;
+        let metadata = DataResponseMetadata::default();
+        // TODO(#1109): Set metadata.data_langid correctly.
         Ok(DataResponse {
-            metadata: DataResponseMetadata {
-                data_langid: req.resource_path.options.langid.clone(),
-            },
+            metadata,
             payload: Some(DataPayload::from_owned(calendar::DatePatternsV1::from(
                 dates,
             ))),