Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft sketch of ResolvedLocaleAdapter #4607

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 24 additions & 0 deletions components/locid/src/langid.rs
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,30 @@ impl LanguageIdentifier {
variants: subtags::Variants::new(),
};

/// Borrows the fields of this [`LanguageIdentifier`] in a tuple.
pub fn as_tuple(
&self,
) -> (
subtags::Language,
Option<subtags::Script>,
Option<subtags::Region>,
&subtags::Variants,
) {
(self.language, self.script, self.region, &self.variants)
}

/// Takes the fields of this [`LanguageIdentifier`] and returns them as a tuple.
pub fn into_tuple(
self,
) -> (
subtags::Language,
Option<subtags::Script>,
Option<subtags::Region>,
subtags::Variants,
) {
(self.language, self.script, self.region, self.variants)
}

/// This is a best-effort operation that performs all available levels of canonicalization.
///
/// At the moment the operation will normalize casing and the separator, but in the future
Expand Down
2 changes: 2 additions & 0 deletions provider/adapters/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
//! - Use the [`either`] module to choose between multiple provider types at runtime.
//! - Use the [`filter`] module to programmatically reject certain data requests.
//! - Use the [`fallback`] module to automatically resolve arbitrary locales for data loading.
//! - Use the [`resolved`] module to determine the supported or resolved locale for a data request.

// https://github.com/unicode-org/icu4x/blob/main/docs/process/boilerplate.md#library-annotations
#![cfg_attr(not(any(test, feature = "std")), no_std)]
Expand All @@ -34,3 +35,4 @@ pub mod fallback;
pub mod filter;
pub mod fork;
mod helpers;
pub mod resolved;
73 changes: 73 additions & 0 deletions provider/adapters/src/resolved.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

use alloc::collections::{BTreeMap, BTreeSet};
use core::cell::RefCell;

use icu_provider::prelude::*;

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
struct ResolvedLocaleInfo {
pub requested_locale: DataLocale,
pub resolved_locale: Option<DataLocale>,
}

/// TODO: Docs
#[derive(Debug)]
pub struct ResolvedLocaleAdapter<P> {
inner: P,
resolved_locales: RefCell<BTreeMap<DataKey, ResolvedLocaleInfo>>,
drop_payloads: bool,
}

impl<P> ResolvedLocaleAdapter<P> {
pub fn into_inner(self) -> P {
self.inner
}

pub fn clear(&mut self) {
self.resolved_locales.borrow_mut().clear()
}

pub fn take_resolved_locale_for_key(&mut self, key: DataKey) -> Option<DataLocale> {
self.resolved_locales
.borrow_mut()
.remove(&key)
.and_then(|info| info.resolved_locale)
}

pub fn take_all_resolved_locales(&mut self) -> BTreeSet<DataLocale> {
let map = self.resolved_locales.take();
map.into_iter()
.filter_map(|(_, info)| info.resolved_locale)
.collect()
}

pub fn saw_last_resort_fallback(&self) -> bool {
self.resolved_locales.borrow().values().any(|info| {
info.resolved_locale
.as_ref()
.map(|l| l.is_langid_und())
.unwrap_or(false)
})
}
}

impl<P: BufferProvider> BufferProvider for ResolvedLocaleAdapter<P> {
fn load_buffer(
&self,
key: DataKey,
req: DataRequest,
) -> Result<DataResponse<BufferMarker>, DataError> {
let mut response = self.inner.load_buffer(key, req)?;
self.resolved_locales.borrow_mut().insert(
key,
ResolvedLocaleInfo {
requested_locale: req.locale.clone(),
resolved_locale: response.metadata.locale.take(),
},
);
Ok(response)
}
}
12 changes: 8 additions & 4 deletions provider/blob/src/blob_data_provider.rs
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,14 @@ impl BufferProvider for BlobDataProvider {
metadata.buffer_format = Some(BufferFormat::Postcard1);
Ok(DataResponse {
metadata,
payload: Some(DataPayload::from_yoked_buffer(
self.data
.try_map_project_cloned(|blob, _| blob.load(key, req))?,
)),
payload: if req.metadata.drop_payload {
None
} else {
Some(DataPayload::from_yoked_buffer(
self.data
.try_map_project_cloned(|blob, _| blob.load(key, req))?,
))
},
})
}
}
Expand Down
24 changes: 16 additions & 8 deletions provider/core/src/any.rs
Original file line number Diff line number Diff line change
Expand Up @@ -437,10 +437,7 @@ where
{
#[inline]
fn load(&self, req: DataRequest) -> Result<DataResponse<M>, DataError> {
self.0
.load_any(M::KEY, req)?
.downcast()
.map_err(|e| e.with_req(M::KEY, req))
self.load_data(M::KEY, req)
}
}

Expand All @@ -454,10 +451,21 @@ where
{
#[inline]
fn load_data(&self, key: DataKey, req: DataRequest) -> Result<DataResponse<M>, DataError> {
self.0
.load_any(key, req)?
.downcast()
.map_err(|e| e.with_req(key, req))
let any_response = AnyProvider::load_any(self.0, key, req)?;
Ok(DataResponse {
metadata: any_response.metadata,
payload: any_response
.payload
.and_then(|p| {
if req.metadata.drop_payload {
None
} else {
Some(p.downcast())
}
})
.transpose()
.map_err(|e| e.with_req(key, req))?,
})
}
}

Expand Down
37 changes: 36 additions & 1 deletion provider/core/src/request.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ impl fmt::Display for DataRequest<'_> {
pub struct DataRequestMetadata {
/// Silent requests do not log errors. This can be used for exploratory querying, such as fallbacks.
pub silent: bool,
/// Whether to drop the payload from the [`DataResponse`](crate::DataResponse). This can be used
/// for exploratory queries where the returned data is not of interest.
pub drop_payload: bool,
}

/// A locale type optimized for use in fallbacking and the ICU4X data pipeline.
Expand Down Expand Up @@ -246,6 +249,22 @@ impl FromStr for DataLocale {
}
}

impl PartialOrd for DataLocale {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}

impl Ord for DataLocale {
fn cmp(&self, other: &Self) -> Ordering {
self.langid
.as_tuple()
Comment on lines +260 to +261
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note to self: we shouldn't implement this, even on DataLocale, due to the same reasons as discussed in #1215. We should instead implement Ord only on the private ResolvedLocaleInfo type.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

.cmp(&other.langid.as_tuple())
.then_with(|| self.keywords.cmp(&other.keywords))
.then_with(|| self.aux.cmp(&other.aux))
}
}

impl DataLocale {
/// Compare this [`DataLocale`] with BCP-47 bytes.
///
Expand Down Expand Up @@ -755,7 +774,7 @@ impl DataLocale {
/// ```
///
/// [`Keywords`]: unicode_ext::Keywords
#[derive(Debug, PartialEq, Clone, Eq, Hash)]
#[derive(Debug, PartialEq, Clone, Eq, Hash, PartialOrd, Ord)]
#[cfg(feature = "experimental")]
pub struct AuxiliaryKeys {
value: AuxiliaryKeysInner,
Expand Down Expand Up @@ -809,6 +828,22 @@ impl Hash for AuxiliaryKeysInner {
}
}

#[cfg(feature = "experimental")]
impl PartialOrd for AuxiliaryKeysInner {
#[inline]
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
self.deref().partial_cmp(other.deref())
}
}

#[cfg(feature = "experimental")]
impl Ord for AuxiliaryKeysInner {
#[inline]
fn cmp(&self, other: &Self) -> Ordering {
self.deref().cmp(other.deref())
}
}

#[cfg(feature = "experimental")]
writeable::impl_display_with_writeable!(AuxiliaryKeys);

Expand Down
8 changes: 7 additions & 1 deletion provider/core/src/serde/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,13 @@ where
metadata: buffer_response.metadata,
payload: buffer_response
.payload
.map(|p| p.into_deserialized(buffer_format))
.and_then(|p| {
if req.metadata.drop_payload {
None
} else {
Some(p.into_deserialized(buffer_format))
}
})
.transpose()
.map_err(|e| e.with_req(key, req))?,
})
Expand Down
8 changes: 7 additions & 1 deletion provider/fs/src/fs_data_provider.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,15 @@ impl BufferProvider for FsDataProvider {
if !Path::new(&path).exists() {
return Err(DataErrorKind::MissingLocale.with_req(key, req));
}
let buffer = fs::read(&path).map_err(|e| DataError::from(e).with_path_context(&path))?;
let mut metadata = DataResponseMetadata::default();
metadata.buffer_format = Some(self.manifest.buffer_format);
if req.metadata.drop_payload {
return Ok(DataResponse {
metadata,
payload: None,
});
}
let buffer = fs::read(&path).map_err(|e| DataError::from(e).with_path_context(&path))?;
Ok(DataResponse {
metadata,
payload: Some(DataPayload::from_owned_buffer(buffer.into_boxed_slice())),
Expand Down