diff --git a/.gitignore b/.gitignore index ef9ba415a21..e423997ba57 100644 --- a/.gitignore +++ b/.gitignore @@ -22,9 +22,9 @@ dhat-heap.json /benchmarks # Do not check-in binary file tree test data +provider/datagen/tests/testdata/work_log_bincode provider/testdata/data/bincode provider/testdata/data/postcard -tools/datagen/tests/testdata/work_log_bincode # Ignore irrelevant files that get generated on macOS **/.DS_Store diff --git a/CODEOWNERS b/CODEOWNERS index 93749c22eff..3aa2709d5be 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -26,16 +26,14 @@ ffi/cpp/ @Manishearth ffi/ecma402/ @filmil ffi/wasm/ @Manishearth provider/blob/ @sffc @Manishearth -provider/cldr/ @sffc @nordzilla provider/core/ @sffc @Manishearth +provider/datagen/ @sffc @robertbastian @Manishearth provider/fs/ @sffc provider/macros/ @Manishearth @sffc provider/testdata/ @unicode-org/icu4x-owners -provider/segmenter/ @aethanyc @makotokato tools/benchmark/binsize/ @gnrunge tools/benchmark/macros/ @gregtatum tools/benchmark/memory/ @gregtatum -tools/datagen/ @sffc tools/scripts/ @Manishearth @sffc utils/fixed_decimal/ @sffc utils/litemap/ @Manishearth @sffc diff --git a/Cargo.lock b/Cargo.lock index f9b18edb4c2..c93f27af08d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,9 +580,9 @@ dependencies = [ [[package]] name = "dirs-sys" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", @@ -1136,27 +1136,49 @@ version = "0.5.0" dependencies = [ "clap", "dhat 0.3.0", + "dirs", + "displaydoc", "eyre", "futures", + "icu_calendar", + "icu_casemapping", + "icu_codepointtrie", + "icu_datetime", + "icu_decimal", + "icu_list", + "icu_locale_canonicalizer", "icu_locid", + "icu_locid_macros", + "icu_plurals", "icu_properties", "icu_provider", "icu_provider_adapters", "icu_provider_blob", - "icu_provider_cldr", "icu_provider_fs", - "icu_provider_segmenter", - "icu_provider_uprops", "icu_segmenter", "icu_testdata", + "icu_uniset", + "itertools", + "json", "litemap", "log", + "mktemp", "rayon", "reqwest", "serde", + "serde-aux", + "serde-tuple-vec-map", + "serde_json", "simple_logger", + "smallstr", + "smallvec", + "tinystr 0.5.0", "tokio", + "toml", + "unzip", + "urlencoding", "writeable", + "zerovec", ] [[package]] @@ -1363,44 +1385,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_provider_cldr" -version = "0.5.0" -dependencies = [ - "dirs", - "displaydoc", - "icu_calendar", - "icu_datetime", - "icu_decimal", - "icu_list", - "icu_locale_canonicalizer", - "icu_locid", - "icu_locid_macros", - "icu_plurals", - "icu_properties", - "icu_provider", - "icu_provider_adapters", - "icu_provider_uprops", - "icu_testdata", - "itertools", - "json", - "litemap", - "log", - "mktemp", - "reqwest", - "serde", - "serde-aux", - "serde-tuple-vec-map", - "serde_json", - "smallstr", - "smallvec", - "tinystr 0.5.0", - "unzip", - "urlencoding", - "writeable", - "zerovec", -] - [[package]] name = "icu_provider_fs" version = "0.5.0" @@ -1433,43 +1417,6 @@ dependencies = [ "syn", ] -[[package]] -name = "icu_provider_segmenter" -version = "0.1.0" -dependencies = [ - "icu_codepointtrie", - "icu_properties", - "icu_provider", - "icu_provider_adapters", - "icu_provider_uprops", - "icu_segmenter", - "icu_testdata", - "serde", - "toml", - "zerovec", -] - -[[package]] -name = "icu_provider_uprops" -version = "0.5.0" -dependencies = [ - "displaydoc", - "eyre", - "icu_casemapping", - "icu_codepointtrie", - "icu_locid", - "icu_properties", - "icu_provider", - "icu_provider_adapters", - "icu_testdata", - "icu_uniset", - "log", - "serde", - "tinystr 0.4.11", - "toml", - "zerovec", -] - [[package]] name = "icu_segmenter" version = "0.1.0" @@ -2165,21 +2112,22 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55" dependencies = [ "getrandom", "redox_syscall", + "thiserror", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 097873bc4ae..610180d35e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,17 +27,14 @@ members = [ "ffi/ecma402", "provider/adapters", "provider/blob", - "provider/cldr", "provider/core", + "provider/datagen", "provider/fs", "provider/macros", - "provider/segmenter", "provider/testdata", - "provider/uprops", "tools/benchmark/macros", "tools/benchmark/memory", "tools/benchmark/binsize", - "tools/datagen", "utils/codepointtrie", "utils/deduplicating_array", "utils/fixed_decimal", diff --git a/docs/tutorials/writing_a_new_data_struct.md b/docs/tutorials/writing_a_new_data_struct.md index 6f216a18a28..507e527fc48 100644 --- a/docs/tutorials/writing_a_new_data_struct.md +++ b/docs/tutorials/writing_a_new_data_struct.md @@ -55,11 +55,11 @@ Although they may share common code, source data providers are implemented speci Examples of source data providers include: -- [`CldrJsonDataProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_provider_cldr/transform/struct.CldrJsonDataProvider.html#) - - [`NumbersProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_provider_cldr/transform/struct.NumbersProvider.html) - - [`PluralsProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_provider_cldr/transform/struct.PluralsProvider.html) - - [`DateSymbolsProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_provider_cldr/transform/struct.DateSymbolsProvider.html) - - [… more examples](https://unicode-org.github.io/icu4x-docs/doc/icu_provider_cldr/transform/index.html) +- [`CldrJsonDataProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_datagen/cldr/transform/struct.CldrJsonDataProvider.html#) + - [`NumbersProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_datagen/cldr/transform/struct.NumbersProvider.html) + - [`PluralsProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_datagen/cldr/transform/struct.PluralsProvider.html) + - [`DateSymbolsProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_datagen/cldr/transform/struct.DateSymbolsProvider.html) + - [… more examples](https://unicode-org.github.io/icu4x-docs/doc/icu_datagen/cldr/transform/index.html) - `BinaryPropertyUnicodeSetDataProvider` - [`HelloWorldProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_provider/hello_world/struct.HelloWorldProvider.html) diff --git a/provider/blob/README.md b/provider/blob/README.md index d1bff1efb6d..e3a687977f3 100644 --- a/provider/blob/README.md +++ b/provider/blob/README.md @@ -31,7 +31,7 @@ For more examples, see the specific data providers. [`ICU4X`]: ../icu/index.html [`BufferProvider`]: icu_provider::BufferProvider -[`icu4x-datagen`]: https://github.com/unicode-org/icu4x/tree/main/tools/datagen#readme +[`icu4x-datagen`]: https://github.com/unicode-org/icu4x/tree/main/provider/datagen#readme ## More Information diff --git a/provider/blob/src/lib.rs b/provider/blob/src/lib.rs index a74514f08cd..3c445d9238b 100644 --- a/provider/blob/src/lib.rs +++ b/provider/blob/src/lib.rs @@ -33,7 +33,7 @@ //! //! [`ICU4X`]: ../icu/index.html //! [`BufferProvider`]: icu_provider::BufferProvider -//! [`icu4x-datagen`]: https://github.com/unicode-org/icu4x/tree/main/tools/datagen#readme +//! [`icu4x-datagen`]: https://github.com/unicode-org/icu4x/tree/main/provider/datagen#readme // https://github.com/unicode-org/icu4x/blob/main/docs/process/boilerplate.md#library-annotations #![cfg_attr(not(any(test, feature = "std")), no_std)] diff --git a/provider/cldr/README.md b/provider/cldr/README.md deleted file mode 100644 index e532b6c151c..00000000000 --- a/provider/cldr/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# icu_provider_cldr [![crates.io](https://img.shields.io/crates/v/icu_provider_cldr)](https://crates.io/crates/icu_provider_cldr) - -`icu_provider_cldr` contains implementations of the [`ICU4X`] [data provider] interface -based on the JSON files shipped by CLDR. Create a [`CldrPaths`] and then pass it into -[`create_exportable_provider`]. - -This crate contains two implementations of [`CldrPaths`]: - -- [`CldrPathsLocal`] points to local copies of the CLDR JSON repositories. -- [`CldrAllInOneDownloader`](download::CldrAllInOneDownloader) downloads and caches the - CLDR JSON repositories. Requires the "download" feature. - -**Important:** This data provider implementation is not optimized for production use. -It is much more efficient if you use [`FsDataProvider`] instead. - -[`ICU4X`]: ../icu/index.html -[data provider]: icu_provider -[`FsDataProvider`]: ../icu_provider_fs/struct.FsDataProvider.html - -## More Information - -For more information on development, authorship, contributing etc. please visit [`ICU4X home page`](https://github.com/unicode-org/icu4x). diff --git a/provider/core/README.md b/provider/core/README.md index f0c33c861a8..de01efec26b 100644 --- a/provider/core/README.md +++ b/provider/core/README.md @@ -111,7 +111,7 @@ This trait is normally implemented using the [`impl_dyn_provider!`] macro using [`icu_provider_adapters`]: ../icu_provider_adapters/index.html [`as_downcasting()`]: AsDowncastingAnyProvider::as_downcasting [`as_deserializing()`]: AsDeserializingBufferProvider::as_deserializing -[`CldrJsonDataProvider`]: ../icu_provider_cldr/struct.CldrJsonDataProvider.html +[`CldrJsonDataProvider`]: ../icu_datagen/cldr/struct.CldrJsonDataProvider.html [`FsDataProvider`]: ../icu_provider_fs/struct.FsDataProvider.html [`BlobDataProvider`]: ../icu_provider_blob/struct.BlobDataProvider.html diff --git a/provider/core/src/lib.rs b/provider/core/src/lib.rs index a1d92af776d..c6bbeecc2ea 100644 --- a/provider/core/src/lib.rs +++ b/provider/core/src/lib.rs @@ -113,7 +113,7 @@ //! [`icu_provider_adapters`]: ../icu_provider_adapters/index.html //! [`as_downcasting()`]: AsDowncastingAnyProvider::as_downcasting //! [`as_deserializing()`]: AsDeserializingBufferProvider::as_deserializing -//! [`CldrJsonDataProvider`]: ../icu_provider_cldr/struct.CldrJsonDataProvider.html +//! [`CldrJsonDataProvider`]: ../icu_datagen/cldr/struct.CldrJsonDataProvider.html //! [`FsDataProvider`]: ../icu_provider_fs/struct.FsDataProvider.html //! [`BlobDataProvider`]: ../icu_provider_blob/struct.BlobDataProvider.html diff --git a/provider/cldr/Cargo.toml b/provider/datagen/Cargo.toml similarity index 54% rename from provider/cldr/Cargo.toml rename to provider/datagen/Cargo.toml index 6716697df3f..5362bbe833c 100644 --- a/provider/cldr/Cargo.toml +++ b/provider/datagen/Cargo.toml @@ -3,8 +3,8 @@ # (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). [package] -name = "icu_provider_cldr" -description = "Data provider that reads from a CLDR JSON data source" +name = "icu_datagen" +description = "Generate data for ICU4X DataProvider" version = "0.5.0" authors = ["The ICU4X Project Developers"] edition = "2018" @@ -20,60 +20,89 @@ include = [ "tests/**/*", "Cargo.toml", "LICENSE", - "README.md" + "README.md", ] [package.metadata.cargo-all-features] # Omit most optional dependency features from permutation testing skip_optional_dependencies = true -# Bench feature gets tested separately and is only relevant for CI -denylist = ["bench"] -extra_features = [ - "log", -] [package.metadata.docs.rs] all-features = true [dependencies] -icu_provider = { version = "0.5", path = "../../provider/core", features = ["serialize", "datagen"] } -icu_provider_adapters = { path = "../../provider/adapters" } -icu_locid = { version = "0.5", path = "../../components/locid" } -icu_plurals = { version = "0.5", path = "../../components/plurals", features = ["datagen"] } +clap = "2.33" +dhat = "0.3.0" +displaydoc = { version = "0.2.3", default-features = false } +eyre = "0.6" +futures = "0.3" +icu_calendar = { version = "0.5", path = "../../components/calendar", features = ["datagen"] } +icu_casemapping = { version = "0.1", path = "../../experimental/casemapping", features = ["provider_serde", "provider_transform_internals"], optional = true } +icu_codepointtrie = { version = "0.3.3", path = "../../utils/codepointtrie", features = ["serialize"] } icu_datetime = { version = "0.5", path = "../../components/datetime", features = ["datagen"] } -icu_locale_canonicalizer = { version = "0.5", path = "../../components/locale_canonicalizer", features = ["datagen"] } icu_decimal = { version = "0.5", path = "../../components/decimal", features = ["datagen"] } -icu_calendar = { version = "0.5", path = "../../components/calendar", features = ["datagen"] } icu_list = { version = "0.5", path = "../../components/list", features = ["datagen"]} +icu_locale_canonicalizer = { version = "0.5", path = "../../components/locale_canonicalizer", features = ["datagen"] } +icu_locid = { version = "0.5", path = "../../components/locid", features = ["std"]} +icu_locid_macros = { version = "0.5", path = "../../components/locid/macros" } +icu_plurals = { version = "0.5", path = "../../components/plurals", features = ["datagen"] } +icu_properties = { version = "0.5", path = "../../components/properties", features = ["std", "datagen"]} +icu_provider = { version = "0.5", path = "../../provider/core", features = ["std", "log_error_context"]} +icu_provider_adapters = { path = "../../provider/adapters", features = ["datagen"] } +icu_provider_blob = { version = "0.5", path = "../../provider/blob", features = ["export"] } +icu_provider_fs = { version = "0.5", path = "../../provider/fs", features = ["export"] } +icu_segmenter = { version = "0.1", path = "../../experimental/segmenter", features = ["datagen"] } +icu_testdata = { version = "0.5", path = "../../provider/testdata", features = ["metadata"] } +icu_uniset = { version = "0.4.1", path = "../../utils/uniset", features = ["serialize"] } itertools = "0.10" json = "0.12" -litemap = { version = "0.3.0", path = "../../utils/litemap/" } +litemap = { version = "0.3.0", path = "../../utils/litemap" } +log = { version = "0.4" } +rayon = "1.5" serde = { version = "1.0", default-features = false, features = ["derive", "alloc"] } -serde-aux = "2.1.1" serde_json = { version = "1.0", default-features = false, features = ["alloc"] } +serde-aux = "2.1.1" serde-tuple-vec-map = "1.0" +simple_logger = "1.12" smallstr = { version = "0.2", features = ["serde"] } smallvec = "1.6" tinystr = { path = "../../utils/tinystr", version = "0.5.0", features = ["alloc", "serde", "zerovec"], default-features = false } -displaydoc = { version = "0.2.3", default-features = false } -icu_locid_macros = { version = "0.5", path = "../../components/locid/macros" } -icu_provider_uprops = { version = "0.5", path = "../../provider/uprops", features = ["log"] } -icu_properties = { version = "0.5", path = "../../components/properties", features = ["datagen", "std"] } -zerovec = { version = "0.6", path = "../../utils/zerovec"} +tokio = { version = "1.5", features = ["rt-multi-thread", "macros", "fs"] } +toml = { version = "0.5" } +writeable = { version = "0.3", path = "../../utils/writeable" } +zerovec = { version = "0.6", path = "../../utils/zerovec", features = ["serde", "yoke"] } # Dependencies for the download feature urlencoding = { version = "1.1", optional = true } -reqwest = { version = "0.11", features = ["blocking"], optional = true } +reqwest = { version = "0.11", features = ["json", "stream", "gzip", "blocking"], optional = true } unzip = { version = "0.1", optional = true } dirs = { version = "3.0", optional = true } -log = { version = "0.4", optional = true } [dev-dependencies] mktemp = "0.4" -icu_locid_macros = { version = "0.5", path = "../../components/locid/macros" } +icu_locid = { version = "0.5", path = "../../components/locid" } icu_testdata = { version = "0.5", path = "../../provider/testdata" } writeable = { version = "0.3", path = "../../utils/writeable" } [features] +experimental = [] +casemapping = ["icu_casemapping"] # Automatically download CLDR data from a host -download = ["urlencoding", "reqwest", "unzip", "dirs", "log"] +download = ["urlencoding", "reqwest", "unzip", "dirs"] +# TODO(#1739): Disable the "download" feature by default +default = ["casemapping", "download"] + +[[bin]] +name = "icu4x-datagen" +path = "src/bin/datagen.rs" +# TODO(#1739): Make it such that icu4x-datagen does not require the download feature +required-features = ["download"] + +[[bin]] +name = "icu4x-testdata-download" +path = "src/bin/testdata-download.rs" +required-features = ["download"] + +[[bin]] +name = "icu4x-verify-zero-copy" +path = "src/bin/verify-zero-copy.rs" diff --git a/provider/cldr/LICENSE b/provider/datagen/LICENSE similarity index 100% rename from provider/cldr/LICENSE rename to provider/datagen/LICENSE diff --git a/tools/datagen/README.md b/provider/datagen/README.md similarity index 100% rename from tools/datagen/README.md rename to provider/datagen/README.md diff --git a/provider/segmenter/data/grapheme.toml b/provider/datagen/data/grapheme.toml similarity index 100% rename from provider/segmenter/data/grapheme.toml rename to provider/datagen/data/grapheme.toml diff --git a/provider/segmenter/data/line.toml b/provider/datagen/data/line.toml similarity index 100% rename from provider/segmenter/data/line.toml rename to provider/datagen/data/line.toml diff --git a/provider/segmenter/data/sentence.toml b/provider/datagen/data/sentence.toml similarity index 100% rename from provider/segmenter/data/sentence.toml rename to provider/datagen/data/sentence.toml diff --git a/provider/segmenter/data/word.toml b/provider/datagen/data/word.toml similarity index 100% rename from provider/segmenter/data/word.toml rename to provider/datagen/data/word.toml diff --git a/tools/datagen/src/bin/datagen.rs b/provider/datagen/src/bin/datagen.rs similarity index 97% rename from tools/datagen/src/bin/datagen.rs rename to provider/datagen/src/bin/datagen.rs index e848769fc05..861dc3ff3b4 100644 --- a/tools/datagen/src/bin/datagen.rs +++ b/provider/datagen/src/bin/datagen.rs @@ -4,7 +4,12 @@ use clap::{App, Arg, ArgGroup, ArgMatches}; use eyre::WrapErr; +use icu_datagen::cldr; +use icu_datagen::cldr::download::CldrAllInOneDownloader; +use icu_datagen::cldr::CldrPathsAllInOne; use icu_datagen::get_all_keys; +use icu_datagen::segmenter; +use icu_datagen::uprops; use icu_locid::LanguageIdentifier; use icu_provider::datagen::IterableDynProvider; use icu_provider::export::DataExporter; @@ -14,8 +19,6 @@ use icu_provider::serde::SerializeMarker; use icu_provider_adapters::filter::Filterable; use icu_provider_adapters::fork::by_key::MultiForkByKeyProvider; use icu_provider_blob::export::BlobExporter; -use icu_provider_cldr::download::CldrAllInOneDownloader; -use icu_provider_cldr::CldrPathsAllInOne; use icu_provider_fs::export::fs_exporter; use icu_provider_fs::export::serializers; use icu_provider_fs::export::FilesystemExporter; @@ -324,18 +327,16 @@ fn main() -> eyre::Result<()> { eyre::bail!("Value for --uprops-root must be specified",) }; - let segmenter_data_root = icu_provider_segmenter::segmenter_data_root(); + let segmenter_data_root = icu_datagen::segmenter::segmenter_data_root(); Box::new(MultiForkByKeyProvider { providers: vec![ - Box::new(icu_provider_cldr::create_exportable_provider( + Box::new(cldr::create_exportable_provider( cldr_paths.as_ref(), uprops_root.clone(), )?), - Box::new(icu_provider_uprops::create_exportable_provider( - &uprops_root, - )?), - Box::new(icu_provider_segmenter::create_exportable_provider( + Box::new(uprops::create_exportable_provider(&uprops_root)?), + Box::new(segmenter::create_exportable_provider( &segmenter_data_root, &uprops_root, )?), diff --git a/tools/datagen/src/bin/testdata-download.rs b/provider/datagen/src/bin/testdata-download.rs similarity index 100% rename from tools/datagen/src/bin/testdata-download.rs rename to provider/datagen/src/bin/testdata-download.rs diff --git a/tools/datagen/src/bin/verify-zero-copy.rs b/provider/datagen/src/bin/verify-zero-copy.rs similarity index 99% rename from tools/datagen/src/bin/verify-zero-copy.rs rename to provider/datagen/src/bin/verify-zero-copy.rs index e21c80a6ba5..25e6f4f4eae 100644 --- a/tools/datagen/src/bin/verify-zero-copy.rs +++ b/provider/datagen/src/bin/verify-zero-copy.rs @@ -10,8 +10,8 @@ use icu_provider_adapters::filter::Filterable; use icu_provider::prelude::*; +use icu_datagen::cldr::CldrPathsAllInOne; use icu_provider_blob::BlobDataProvider; -use icu_provider_cldr::CldrPathsAllInOne; use litemap::LiteMap; use simple_logger::SimpleLogger; use std::borrow::Cow; @@ -178,7 +178,7 @@ fn main() -> eyre::Result<()> { .to_string(), }); - let segmenter_data_root = icu_provider_segmenter::segmenter_data_root(); + let segmenter_data_root = icu_datagen::segmenter::segmenter_data_root(); let converter = icu_datagen::get_registry(&*cldr_paths, &uprops_root, &segmenter_data_root)?; diff --git a/provider/cldr/src/cldr_paths.rs b/provider/datagen/src/cldr/cldr_paths.rs similarity index 97% rename from provider/cldr/src/cldr_paths.rs rename to provider/datagen/src/cldr/cldr_paths.rs index f0c4a812c96..789564c7627 100644 --- a/provider/cldr/src/cldr_paths.rs +++ b/provider/datagen/src/cldr/cldr_paths.rs @@ -2,7 +2,7 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::error::{Error, MissingSourceError}; +use super::error::{Error, MissingSourceError}; use std::default::Default; use std::path::PathBuf; @@ -77,8 +77,8 @@ pub trait CldrPaths: std::fmt::Debug + Sync { /// # Examples /// /// ``` -/// use icu_provider_cldr::CldrPaths; -/// use icu_provider_cldr::CldrPathsLocal; +/// use icu_datagen::cldr::CldrPaths; +/// use icu_datagen::cldr::CldrPathsLocal; /// use std::path::PathBuf; /// /// let mut paths = CldrPathsLocal::default(); @@ -150,8 +150,8 @@ impl Default for CldrPathsLocal { /// # Examples /// /// ``` -/// use icu_provider_cldr::CldrPaths; -/// use icu_provider_cldr::CldrPathsAllInOne; +/// use icu_datagen::cldr::CldrPaths; +/// use icu_datagen::cldr::CldrPathsAllInOne; /// use std::path::PathBuf; /// /// let paths = CldrPathsAllInOne { diff --git a/provider/cldr/src/cldr_serde/aliases.rs b/provider/datagen/src/cldr/cldr_serde/aliases.rs similarity index 100% rename from provider/cldr/src/cldr_serde/aliases.rs rename to provider/datagen/src/cldr/cldr_serde/aliases.rs diff --git a/provider/cldr/src/cldr_serde/ca.rs b/provider/datagen/src/cldr/cldr_serde/ca.rs similarity index 100% rename from provider/cldr/src/cldr_serde/ca.rs rename to provider/datagen/src/cldr/cldr_serde/ca.rs diff --git a/provider/cldr/src/cldr_serde/japanese.rs b/provider/datagen/src/cldr/cldr_serde/japanese.rs similarity index 100% rename from provider/cldr/src/cldr_serde/japanese.rs rename to provider/datagen/src/cldr/cldr_serde/japanese.rs diff --git a/provider/cldr/src/cldr_serde/likely_subtags.rs b/provider/datagen/src/cldr/cldr_serde/likely_subtags.rs similarity index 100% rename from provider/cldr/src/cldr_serde/likely_subtags.rs rename to provider/datagen/src/cldr/cldr_serde/likely_subtags.rs diff --git a/provider/cldr/src/cldr_serde/list_patterns.rs b/provider/datagen/src/cldr/cldr_serde/list_patterns.rs similarity index 100% rename from provider/cldr/src/cldr_serde/list_patterns.rs rename to provider/datagen/src/cldr/cldr_serde/list_patterns.rs diff --git a/provider/cldr/src/cldr_serde/mod.rs b/provider/datagen/src/cldr/cldr_serde/mod.rs similarity index 100% rename from provider/cldr/src/cldr_serde/mod.rs rename to provider/datagen/src/cldr/cldr_serde/mod.rs diff --git a/provider/cldr/src/cldr_serde/numbering_systems.rs b/provider/datagen/src/cldr/cldr_serde/numbering_systems.rs similarity index 100% rename from provider/cldr/src/cldr_serde/numbering_systems.rs rename to provider/datagen/src/cldr/cldr_serde/numbering_systems.rs diff --git a/provider/cldr/src/cldr_serde/numbers.rs b/provider/datagen/src/cldr/cldr_serde/numbers.rs similarity index 100% rename from provider/cldr/src/cldr_serde/numbers.rs rename to provider/datagen/src/cldr/cldr_serde/numbers.rs diff --git a/provider/cldr/src/cldr_serde/plurals.rs b/provider/datagen/src/cldr/cldr_serde/plurals.rs similarity index 100% rename from provider/cldr/src/cldr_serde/plurals.rs rename to provider/datagen/src/cldr/cldr_serde/plurals.rs diff --git a/provider/cldr/src/cldr_serde/time_zone_names.rs b/provider/datagen/src/cldr/cldr_serde/time_zone_names.rs similarity index 100% rename from provider/cldr/src/cldr_serde/time_zone_names.rs rename to provider/datagen/src/cldr/cldr_serde/time_zone_names.rs diff --git a/provider/cldr/src/cldr_serde/week_data.rs b/provider/datagen/src/cldr/cldr_serde/week_data.rs similarity index 100% rename from provider/cldr/src/cldr_serde/week_data.rs rename to provider/datagen/src/cldr/cldr_serde/week_data.rs diff --git a/provider/cldr/src/download/cldr_allinone.rs b/provider/datagen/src/cldr/download/cldr_allinone.rs similarity index 95% rename from provider/cldr/src/download/cldr_allinone.rs rename to provider/datagen/src/cldr/download/cldr_allinone.rs index 5ce95730964..54b2c179cbc 100644 --- a/provider/cldr/src/download/cldr_allinone.rs +++ b/provider/datagen/src/cldr/download/cldr_allinone.rs @@ -7,7 +7,7 @@ use super::error::Error; use super::io_util; -use crate::CldrPathsAllInOne; +use crate::cldr::CldrPathsAllInOne; use std::path::PathBuf; /// Implementation of CldrPaths that downloads CLDR data directories on demand. @@ -20,8 +20,8 @@ use std::path::PathBuf; /// /// ```no_run because we don't want to download /// use std::boxed::Box; -/// use icu_provider_cldr::CldrPaths; -/// use icu_provider_cldr::download::CldrAllInOneDownloader; +/// use icu_datagen::cldr::CldrPaths; +/// use icu_datagen::cldr::download::CldrAllInOneDownloader; /// let downloader = CldrAllInOneDownloader::try_new_from_github("38.1.0", "modern") /// .expect("Cache directory not found"); diff --git a/provider/cldr/src/download/error.rs b/provider/datagen/src/cldr/download/error.rs similarity index 100% rename from provider/cldr/src/download/error.rs rename to provider/datagen/src/cldr/download/error.rs diff --git a/provider/cldr/src/download/io_util.rs b/provider/datagen/src/cldr/download/io_util.rs similarity index 100% rename from provider/cldr/src/download/io_util.rs rename to provider/datagen/src/cldr/download/io_util.rs diff --git a/provider/cldr/src/download/mod.rs b/provider/datagen/src/cldr/download/mod.rs similarity index 100% rename from provider/cldr/src/download/mod.rs rename to provider/datagen/src/cldr/download/mod.rs diff --git a/provider/cldr/src/error.rs b/provider/datagen/src/cldr/error.rs similarity index 99% rename from provider/cldr/src/error.rs rename to provider/datagen/src/cldr/error.rs index 695c736d032..8120f230efc 100644 --- a/provider/cldr/src/error.rs +++ b/provider/datagen/src/cldr/error.rs @@ -8,7 +8,7 @@ use icu_provider::DataError; use std::path::{Path, PathBuf}; #[cfg(feature = "download")] -use crate::download; +use crate::cldr::download; #[non_exhaustive] #[derive(Display, Debug)] diff --git a/provider/cldr/src/lib.rs b/provider/datagen/src/cldr/mod.rs similarity index 95% rename from provider/cldr/src/lib.rs rename to provider/datagen/src/cldr/mod.rs index 0f8b087f216..3f3cca8c0fe 100644 --- a/provider/cldr/src/lib.rs +++ b/provider/datagen/src/cldr/mod.rs @@ -2,7 +2,7 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -//! `icu_provider_cldr` contains implementations of the [`ICU4X`] [data provider] interface +//! `icu_datagen::cldr` contains implementations of the [`ICU4X`] [data provider] interface //! based on the JSON files shipped by CLDR. Create a [`CldrPaths`] and then pass it into //! [`create_exportable_provider`]. //! @@ -12,12 +12,11 @@ //! - [`CldrAllInOneDownloader`](download::CldrAllInOneDownloader) downloads and caches the //! CLDR JSON repositories. Requires the "download" feature. //! -//! **Important:** This data provider implementation is not optimized for production use. -//! It is much more efficient if you use [`FsDataProvider`] instead. +//! **Important:** This data provider implementation is not optimized +//! for production use. Read more in the [data provider] docs. //! //! [`ICU4X`]: ../icu/index.html //! [data provider]: icu_provider -//! [`FsDataProvider`]: ../icu_provider_fs/struct.FsDataProvider.html mod cldr_paths; mod cldr_serde; diff --git a/provider/cldr/src/reader.rs b/provider/datagen/src/cldr/reader.rs similarity index 97% rename from provider/cldr/src/reader.rs rename to provider/datagen/src/cldr/reader.rs index c958d498c6d..a8e529046ed 100644 --- a/provider/cldr/src/reader.rs +++ b/provider/datagen/src/cldr/reader.rs @@ -2,7 +2,7 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::error::Error; +use crate::cldr::error::Error; use icu_locid::LanguageIdentifier; use std::fs; use std::fs::File; @@ -12,7 +12,6 @@ use std::str::FromStr; /// Helper function to open a file and return failures as a crate error. pub fn open_reader(path: &Path) -> Result, Error> { - #[cfg(feature = "log")] log::trace!("Reading: {:?}", path); File::open(&path) .map(BufReader::new) diff --git a/provider/cldr/src/transform/calendar/japanese.rs b/provider/datagen/src/cldr/transform/calendar/japanese.rs similarity index 96% rename from provider/cldr/src/transform/calendar/japanese.rs rename to provider/datagen/src/cldr/transform/calendar/japanese.rs index 6e33959cba0..1c6579bdf60 100644 --- a/provider/cldr/src/transform/calendar/japanese.rs +++ b/provider/datagen/src/cldr/transform/calendar/japanese.rs @@ -2,10 +2,10 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::cldr_serde; -use crate::error::Error; -use crate::reader::open_reader; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::error::Error; +use crate::cldr::reader::open_reader; +use crate::cldr::CldrPaths; use icu_calendar::provider::*; use icu_locid_macros::langid; use icu_provider::datagen::IterableResourceProvider; @@ -152,11 +152,11 @@ impl ResourceProvider for JapaneseErasProvider { return Err(DataError::custom( "Era data has changed! This can be for two reasons: Either the CLDR locale data for Japanese eras has \ changed in an incompatible way, or there is a new Japanese era. Please comment out the integrity \ - check in icu_provider_cldr's japanese.rs and inspect the update to japanese@1.json (resource key `calendar/japanese`) \ + check in icu_datagen's japanese.rs and inspect the update to japanese@1.json (resource key `calendar/japanese`) \ in the generated JSON by rerunning the datagen tool (`cargo make testdata` in the ICU4X repo). \ Rerun with ICU4X_SKIP_JAPANESE_INTEGRITY_CHECK=1 to regenerate testdata properly, and check which situation \ it is. If a new era has been introduced, copy over the new testdata to snapshot-japanese@1.json \ - in icu_provider_cldr. If not, it's likely that japanese.rs in icu_provider_cldr will need \ + in icu_datagen. If not, it's likely that japanese.rs in icu_datagen will need \ to be updated to handle the data changes. Once done, be sure to regenerate datetime/symbols@1 as well if not \ doing so already" )); diff --git a/provider/cldr/src/transform/calendar/mod.rs b/provider/datagen/src/cldr/transform/calendar/mod.rs similarity index 100% rename from provider/cldr/src/transform/calendar/mod.rs rename to provider/datagen/src/cldr/transform/calendar/mod.rs diff --git a/provider/cldr/src/transform/calendar/snapshot-japanese@1.json b/provider/datagen/src/cldr/transform/calendar/snapshot-japanese@1.json similarity index 100% rename from provider/cldr/src/transform/calendar/snapshot-japanese@1.json rename to provider/datagen/src/cldr/transform/calendar/snapshot-japanese@1.json diff --git a/provider/cldr/src/transform/datetime/common.rs b/provider/datagen/src/cldr/transform/datetime/common.rs similarity index 89% rename from provider/cldr/src/transform/datetime/common.rs rename to provider/datagen/src/cldr/transform/datetime/common.rs index 588503ab505..a861a54bc2f 100644 --- a/provider/cldr/src/transform/datetime/common.rs +++ b/provider/datagen/src/cldr/transform/datetime/common.rs @@ -2,10 +2,10 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::cldr_serde; -use crate::error::Error; -use crate::reader::{get_langid_subdirectories, get_langid_subdirectory, open_reader}; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::error::Error; +use crate::cldr::reader::{get_langid_subdirectories, get_langid_subdirectory, open_reader}; +use crate::cldr::CldrPaths; use icu_provider::prelude::*; use std::convert::TryFrom; @@ -81,9 +81,11 @@ impl CommonDateProvider { self.paths .iter() .find(|(_, cal, _)| cal == &"gregory") - .ok_or(Error::MissingSource(crate::error::MissingSourceError { - src: "cldr-dates/gregory", - }))?; + .ok_or(Error::MissingSource( + crate::cldr::error::MissingSourceError { + src: "cldr-dates/gregory", + }, + ))?; let mut r = Vec::new(); for (_, cal, path) in &self.paths { diff --git a/provider/cldr/src/transform/datetime/mod.rs b/provider/datagen/src/cldr/transform/datetime/mod.rs similarity index 100% rename from provider/cldr/src/transform/datetime/mod.rs rename to provider/datagen/src/cldr/transform/datetime/mod.rs diff --git a/provider/cldr/src/transform/datetime/patterns.rs b/provider/datagen/src/cldr/transform/datetime/patterns.rs similarity index 98% rename from provider/cldr/src/transform/datetime/patterns.rs rename to provider/datagen/src/cldr/transform/datetime/patterns.rs index 763874af06d..8766bdb2940 100644 --- a/provider/cldr/src/transform/datetime/patterns.rs +++ b/provider/datagen/src/cldr/transform/datetime/patterns.rs @@ -4,10 +4,10 @@ use super::common::CommonDateProvider; -use crate::error::Error; +use crate::cldr::error::Error; -use crate::cldr_serde; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::CldrPaths; use icu_datetime::pattern; use icu_datetime::pattern::CoarseHourCycle; use icu_datetime::provider::calendar::*; @@ -267,7 +267,7 @@ impl From<&cldr_serde::ca::Dates> for DatePatternsV1<'_> { fn test_basic() { use icu_locid_macros::langid; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = DatePatternsProvider::try_from(&cldr_paths as &dyn CldrPaths) .expect("Failed to retrieve provider"); @@ -290,7 +290,7 @@ fn test_basic() { fn test_with_numbering_system() { use icu_locid_macros::langid; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = DatePatternsProvider::try_from(&cldr_paths as &dyn CldrPaths) .expect("Failed to retrieve provider"); diff --git a/provider/cldr/src/transform/datetime/skeletons.rs b/provider/datagen/src/cldr/transform/datetime/skeletons.rs similarity index 98% rename from provider/cldr/src/transform/datetime/skeletons.rs rename to provider/datagen/src/cldr/transform/datetime/skeletons.rs index 90b92af2598..9714ed93c60 100644 --- a/provider/cldr/src/transform/datetime/skeletons.rs +++ b/provider/datagen/src/cldr/transform/datetime/skeletons.rs @@ -3,10 +3,10 @@ // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). use super::common::CommonDateProvider; -use crate::error::Error; +use crate::cldr::error::Error; -use crate::cldr_serde; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::CldrPaths; use icu_datetime::provider::calendar::*; use icu_datetime::skeleton::SkeletonError; @@ -150,7 +150,7 @@ fn test_datetime_skeletons() { use icu_locid_macros::langid; use icu_plurals::PluralCategory; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = DateSkeletonPatternsProvider::try_from(&cldr_paths as &dyn CldrPaths) .expect("Failed to retrieve provider"); diff --git a/provider/cldr/src/transform/datetime/symbols.rs b/provider/datagen/src/cldr/transform/datetime/symbols.rs similarity index 97% rename from provider/cldr/src/transform/datetime/symbols.rs rename to provider/datagen/src/cldr/transform/datetime/symbols.rs index 668c2c26aaf..86e1b920f72 100644 --- a/provider/cldr/src/transform/datetime/symbols.rs +++ b/provider/datagen/src/cldr/transform/datetime/symbols.rs @@ -3,9 +3,9 @@ // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). use super::common::CommonDateProvider; -use crate::cldr_serde; -use crate::error::Error; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::error::Error; +use crate::cldr::CldrPaths; use icu_datetime::provider::calendar::*; use icu_provider::datagen::IterableResourceProvider; use icu_provider::prelude::*; @@ -98,7 +98,7 @@ fn get_era_code_map(calendar: &str) -> BTreeMap { "buddhist" => vec![("0".to_string(), tinystr!(16, "be"))] .into_iter() .collect(), - "japanese" => crate::transform::calendar::japanese::get_era_code_map(), + "japanese" => crate::cldr::transform::calendar::japanese::get_era_code_map(), "coptic" => vec![ ("0".to_string(), tinystr!(16, "bc")), ("1".to_string(), tinystr!(16, "ad")), @@ -231,7 +231,7 @@ symbols_from!( fn test_basic() { use icu_locid_macros::langid; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = DateSymbolsProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap(); let cs_dates: DataPayload = provider @@ -258,7 +258,7 @@ fn test_basic() { fn unalias_contexts() { use icu_locid_macros::langid; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = DateSymbolsProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap(); let cs_dates: DataPayload = provider diff --git a/provider/cldr/src/transform/datetime/week_data.rs b/provider/datagen/src/cldr/transform/datetime/week_data.rs similarity index 97% rename from provider/cldr/src/transform/datetime/week_data.rs rename to provider/datagen/src/cldr/transform/datetime/week_data.rs index 9d18982927b..37fcbb9c5b1 100644 --- a/provider/cldr/src/transform/datetime/week_data.rs +++ b/provider/datagen/src/cldr/transform/datetime/week_data.rs @@ -2,13 +2,13 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::cldr_serde::{ +use crate::cldr::cldr_serde::{ self, week_data::{Territory, DEFAULT_TERRITORY}, }; -use crate::error::Error; -use crate::reader::open_reader; -use crate::CldrPaths; +use crate::cldr::error::Error; +use crate::cldr::reader::open_reader; +use crate::cldr::CldrPaths; use icu_calendar::arithmetic::week_of::CalendarInfo; use icu_datetime::provider::week_data::*; use icu_provider::datagen::IterableResourceProvider; @@ -136,7 +136,7 @@ fn basic_cldr_week_data() { use icu_calendar::types::IsoWeekday; use icu_locid_macros::langid; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = WeekDataProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap(); let default_week_data: DataPayload = provider diff --git a/provider/cldr/src/transform/decimal/decimal_pattern.rs b/provider/datagen/src/cldr/transform/decimal/decimal_pattern.rs similarity index 100% rename from provider/cldr/src/transform/decimal/decimal_pattern.rs rename to provider/datagen/src/cldr/transform/decimal/decimal_pattern.rs diff --git a/provider/cldr/src/transform/decimal/mod.rs b/provider/datagen/src/cldr/transform/decimal/mod.rs similarity index 96% rename from provider/cldr/src/transform/decimal/mod.rs rename to provider/datagen/src/cldr/transform/decimal/mod.rs index e5bef8798f9..789c35185bf 100644 --- a/provider/cldr/src/transform/decimal/mod.rs +++ b/provider/datagen/src/cldr/transform/decimal/mod.rs @@ -2,10 +2,10 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::cldr_serde; -use crate::error::Error; -use crate::reader::{get_langid_subdirectories, get_langid_subdirectory, open_reader}; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::error::Error; +use crate::cldr::reader::{get_langid_subdirectories, get_langid_subdirectory, open_reader}; +use crate::cldr::CldrPaths; use icu_decimal::provider::*; use icu_provider::datagen::IterableResourceProvider; use icu_provider::prelude::*; @@ -193,7 +193,7 @@ impl TryFrom<&cldr_serde::numbers::Numbers> for DecimalSymbolsV1<'static> { fn test_basic() { use icu_locid_macros::langid; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = NumbersProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap(); let ar_decimal: DataPayload = provider diff --git a/provider/cldr/src/transform/list/mod.rs b/provider/datagen/src/cldr/transform/list/mod.rs similarity index 93% rename from provider/cldr/src/transform/list/mod.rs rename to provider/datagen/src/cldr/transform/list/mod.rs index d801b0f0c22..842bd54769a 100644 --- a/provider/cldr/src/transform/list/mod.rs +++ b/provider/datagen/src/cldr/transform/list/mod.rs @@ -2,10 +2,11 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::cldr_serde; -use crate::error::Error; -use crate::reader::{get_langid_subdirectories, get_langid_subdirectory, open_reader}; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::error::Error; +use crate::cldr::reader::{get_langid_subdirectories, get_langid_subdirectory, open_reader}; +use crate::cldr::CldrPaths; +use crate::uprops::EnumeratedPropertyCodePointTrieProvider; use icu_list::provider::*; use icu_locid_macros::langid; use icu_provider::datagen::IterableResourceProvider; @@ -114,11 +115,9 @@ impl>> ResourcePro &format!( "[^{}]", icu_properties::maps::get_script( - &icu_provider_uprops::EnumeratedPropertyCodePointTrieProvider::try_new( - &self.uprops_root - ) - .map_err(|e| DataError::custom("Properties data provider error") - .with_display_context(&e))? + &EnumeratedPropertyCodePointTrieProvider::try_new(&self.uprops_root) + .map_err(|e| DataError::custom("Properties data provider error") + .with_display_context(&e))? ) .map_err(|e| DataError::custom("data for CodePointTrie of Script") .with_display_context(&e))? @@ -178,7 +177,7 @@ mod tests { macro_rules! test { ($langid:literal, $type:ident, $(($input:expr, $output:literal),)+) => { - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = ListProvider::try_from( &cldr_paths as &dyn CldrPaths, icu_testdata::paths::uprops_toml_root()).unwrap(); let f = ListFormatter::$type(langid!($langid), &provider, ListStyle::Wide).unwrap(); diff --git a/provider/cldr/src/transform/locale_canonicalizer/aliases.rs b/provider/datagen/src/cldr/transform/locale_canonicalizer/aliases.rs similarity index 98% rename from provider/cldr/src/transform/locale_canonicalizer/aliases.rs rename to provider/datagen/src/cldr/transform/locale_canonicalizer/aliases.rs index b8553922b41..cc4016eb3c2 100644 --- a/provider/cldr/src/transform/locale_canonicalizer/aliases.rs +++ b/provider/datagen/src/cldr/transform/locale_canonicalizer/aliases.rs @@ -2,10 +2,10 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::cldr_serde; -use crate::error::Error; -use crate::reader::open_reader; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::error::Error; +use crate::cldr::reader::open_reader; +use crate::cldr::CldrPaths; use icu_locale_canonicalizer::provider::*; use icu_locid::{subtags, LanguageIdentifier}; use icu_provider::datagen::IterableResourceProvider; @@ -286,7 +286,7 @@ fn test_rules_cmp() { fn test_basic() { use tinystr::tinystr; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = AliasesProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap(); let data: DataPayload = provider .load_resource(&DataRequest::default()) diff --git a/provider/cldr/src/transform/locale_canonicalizer/likely_subtags.rs b/provider/datagen/src/cldr/transform/locale_canonicalizer/likely_subtags.rs similarity index 96% rename from provider/cldr/src/transform/locale_canonicalizer/likely_subtags.rs rename to provider/datagen/src/cldr/transform/locale_canonicalizer/likely_subtags.rs index dae3c03768c..5c49f51060c 100644 --- a/provider/cldr/src/transform/locale_canonicalizer/likely_subtags.rs +++ b/provider/datagen/src/cldr/transform/locale_canonicalizer/likely_subtags.rs @@ -2,10 +2,10 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::cldr_serde; -use crate::error::Error; -use crate::reader::open_reader; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::error::Error; +use crate::cldr::reader::open_reader; +use crate::cldr::CldrPaths; use icu_locale_canonicalizer::provider::*; use icu_provider::datagen::IterableResourceProvider; use icu_provider::prelude::*; @@ -147,7 +147,7 @@ impl From<&cldr_serde::likely_subtags::Resource> for LikelySubtagsV1 { fn test_basic() { use icu_locid_macros::langid; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = LikelySubtagsProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap(); let result: DataPayload = provider .load_resource(&Default::default()) diff --git a/provider/cldr/src/transform/locale_canonicalizer/mod.rs b/provider/datagen/src/cldr/transform/locale_canonicalizer/mod.rs similarity index 100% rename from provider/cldr/src/transform/locale_canonicalizer/mod.rs rename to provider/datagen/src/cldr/transform/locale_canonicalizer/mod.rs diff --git a/provider/cldr/src/transform/mod.rs b/provider/datagen/src/cldr/transform/mod.rs similarity index 100% rename from provider/cldr/src/transform/mod.rs rename to provider/datagen/src/cldr/transform/mod.rs diff --git a/provider/cldr/src/transform/plurals/mod.rs b/provider/datagen/src/cldr/transform/plurals/mod.rs similarity index 97% rename from provider/cldr/src/transform/plurals/mod.rs rename to provider/datagen/src/cldr/transform/plurals/mod.rs index 6219399f46f..c00916fb49c 100644 --- a/provider/cldr/src/transform/plurals/mod.rs +++ b/provider/datagen/src/cldr/transform/plurals/mod.rs @@ -2,10 +2,10 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::cldr_serde; -use crate::error::Error; -use crate::reader::open_reader; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::error::Error; +use crate::cldr::reader::open_reader; +use crate::cldr::CldrPaths; use icu_plurals::provider::*; use icu_plurals::rules::runtime::ast::Rule; use icu_provider::datagen::IterableResourceProvider; @@ -162,7 +162,7 @@ impl From<&cldr_serde::plurals::LocalePluralRules> for PluralRulesV1<'static> { fn test_basic() { use icu_locid_macros::langid; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = PluralsProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap(); // Spot-check locale 'cs' since it has some interesting entries diff --git a/provider/cldr/src/transform/time_zones/convert.rs b/provider/datagen/src/cldr/transform/time_zones/convert.rs similarity index 99% rename from provider/cldr/src/transform/time_zones/convert.rs rename to provider/datagen/src/cldr/transform/time_zones/convert.rs index b6e992ded45..21127b1edb9 100644 --- a/provider/cldr/src/transform/time_zones/convert.rs +++ b/provider/datagen/src/cldr/transform/time_zones/convert.rs @@ -10,7 +10,7 @@ use std::borrow::Cow; use tinystr::TinyStr8; use zerovec::{ZeroMap, ZeroMap2d}; -use crate::cldr_serde::time_zone_names::*; +use crate::cldr::cldr_serde::time_zone_names::*; /// Performs part 1 of type fallback as specified in the UTS-35 spec for TimeZone Goals: /// https://unicode.org/reports/tr35/tr35-dates.html#Time_Zone_Goals diff --git a/provider/cldr/src/transform/time_zones/mod.rs b/provider/datagen/src/cldr/transform/time_zones/mod.rs similarity index 96% rename from provider/cldr/src/transform/time_zones/mod.rs rename to provider/datagen/src/cldr/transform/time_zones/mod.rs index 6303f3d5694..19251867a97 100644 --- a/provider/cldr/src/transform/time_zones/mod.rs +++ b/provider/datagen/src/cldr/transform/time_zones/mod.rs @@ -2,10 +2,10 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::cldr_serde; -use crate::error::Error; -use crate::reader::{get_langid_subdirectories, get_langid_subdirectory, open_reader}; -use crate::CldrPaths; +use crate::cldr::cldr_serde; +use crate::cldr::error::Error; +use crate::cldr::reader::{get_langid_subdirectories, get_langid_subdirectory, open_reader}; +use crate::cldr::CldrPaths; use icu_datetime::provider::time_zones::*; use icu_locid::LanguageIdentifier; use icu_provider::datagen::IterableResourceProvider; @@ -111,7 +111,7 @@ mod tests { fn basic_cldr_time_zones() { use icu_locid_macros::langid; - let cldr_paths = crate::cldr_paths::for_test(); + let cldr_paths = crate::cldr::cldr_paths::for_test(); let provider = TimeZonesProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap(); let time_zone_formats: DataPayload = provider diff --git a/tools/datagen/src/lib.rs b/provider/datagen/src/lib.rs similarity index 90% rename from tools/datagen/src/lib.rs rename to provider/datagen/src/lib.rs index 4e623f09873..8e684f8b1c9 100644 --- a/tools/datagen/src/lib.rs +++ b/provider/datagen/src/lib.rs @@ -49,17 +49,21 @@ //! --out /tmp/icu4x_data/work_log_json //! ``` +pub mod cldr; +pub mod segmenter; +pub mod uprops; + +use cldr::CldrPaths; use icu_provider::datagen::OmnibusDatagenProvider; use icu_provider::serde::SerializeMarker; use icu_provider::ResourceKey; use icu_provider_adapters::fork::by_key::MultiForkByKeyProvider; -use icu_provider_cldr::CldrPaths; use std::path::Path; /// List of all supported keys pub fn get_all_keys() -> Vec { let mut v = vec![]; - v.extend(icu_provider_cldr::ALL_KEYS); + v.extend(cldr::ALL_KEYS); v.extend(icu_properties::provider::key::ALL_MAP_KEYS); v.extend(icu_properties::provider::key::ALL_SCRIPT_EXTENSIONS_KEYS); v.extend(icu_properties::provider::key::ALL_SET_KEYS); @@ -75,14 +79,12 @@ pub fn get_registry( ) -> Result, eyre::Report> { Ok(MultiForkByKeyProvider { providers: vec![ - Box::new(icu_provider_cldr::create_exportable_provider( + Box::new(cldr::create_exportable_provider( cldr_paths, uprops_root.into(), )?), - Box::new(icu_provider_uprops::create_exportable_provider( - uprops_root, - )?), - Box::new(icu_provider_segmenter::create_exportable_provider( + Box::new(uprops::create_exportable_provider(uprops_root)?), + Box::new(segmenter::create_exportable_provider( segmenter_data_root, uprops_root, )?), diff --git a/tools/datagen/src/main.rs b/provider/datagen/src/main.rs similarity index 100% rename from tools/datagen/src/main.rs rename to provider/datagen/src/main.rs diff --git a/provider/segmenter/src/lib.rs b/provider/datagen/src/segmenter/mod.rs similarity index 78% rename from provider/segmenter/src/lib.rs rename to provider/datagen/src/segmenter/mod.rs index 75eb42b174f..7d804062bb2 100644 --- a/provider/segmenter/src/lib.rs +++ b/provider/datagen/src/segmenter/mod.rs @@ -2,17 +2,17 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -//! `icu_provider_segmenter` contains implementations of the [`ICU4X`] [data provider] interface +//! `icu_datagen::segmenter` contains implementations of the [`ICU4X`] [data provider] interface //! based on Unicode properties and TOML files implementing [Unicode Standard Annex #14][UAX14] and //! [Unicode Standard Annex #29][UAX29] breaking rules. //! -//! **Important:** This provider implementation is not optimized for production use. It is much more -//! efficient if you use [`FsDataProvider`] or [`StaticDataProvider`] instead. +//! **Important:** This data provider implementation is not optimized +//! for production use. Read more in the [data provider] docs. //! //! # Examples //! //! ``` -//! use icu_provider_segmenter::SegmenterRuleProvider; +//! use icu_datagen::segmenter::SegmenterRuleProvider; //! let provider = SegmenterRuleProvider::try_new( //! "/path/to/segmenter/data/directory", //! "/path/to/uprops/data/directory", @@ -28,8 +28,6 @@ //! [data provider]: icu_provider //! [UAX14]: https://www.unicode.org/reports/tr14/ //! [UAX29]: https://www.unicode.org/reports/tr29/ -//! [`FsDataProvider`]: ../icu_provider_fs/struct.FsDataProvider.html -//! [`StaticDataProvider`]: ../icu_provider_blob/struct.StaticDataProvider.html use icu_provider::datagen::OmnibusDatagenProvider; use icu_provider::prelude::*; @@ -38,7 +36,7 @@ use std::path::{Path, PathBuf}; mod transform; -pub use crate::transform::SegmenterRuleProvider; +pub use transform::SegmenterRuleProvider; /// Returns the absolute path to the directory containing the segmenter raw data. pub fn segmenter_data_root() -> PathBuf { diff --git a/provider/segmenter/src/transform.rs b/provider/datagen/src/segmenter/transform.rs similarity index 99% rename from provider/segmenter/src/transform.rs rename to provider/datagen/src/segmenter/transform.rs index 39559093995..e9162a44d46 100644 --- a/provider/segmenter/src/transform.rs +++ b/provider/datagen/src/segmenter/transform.rs @@ -2,6 +2,9 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). +use crate::uprops::{ + BinaryPropertyUnicodeSetDataProvider, EnumeratedPropertyCodePointTrieProvider, +}; use icu_codepointtrie::CodePointTrie; use icu_properties::{ maps, sets, EastAsianWidth, GeneralCategory, GraphemeClusterBreak, LineBreak, SentenceBreak, @@ -9,9 +12,6 @@ use icu_properties::{ }; use icu_provider::datagen::IterableResourceProvider; use icu_provider::prelude::*; -use icu_provider_uprops::{ - BinaryPropertyUnicodeSetDataProvider, EnumeratedPropertyCodePointTrieProvider, -}; use icu_segmenter::symbols::*; use icu_segmenter::*; use std::fs::File; @@ -742,7 +742,7 @@ mod tests { #[test] fn load_grapheme_cluster_data() { let provider = SegmenterRuleProvider::try_new( - &crate::segmenter_data_root(), + &crate::segmenter::segmenter_data_root(), &icu_testdata::paths::uprops_toml_root(), ) .expect("Rule break data should exists!"); diff --git a/provider/uprops/src/bin_uniset.rs b/provider/datagen/src/uprops/bin_uniset.rs similarity index 97% rename from provider/uprops/src/bin_uniset.rs rename to provider/datagen/src/uprops/bin_uniset.rs index 247c788464f..6d1e629dfaf 100644 --- a/provider/uprops/src/bin_uniset.rs +++ b/provider/datagen/src/uprops/bin_uniset.rs @@ -2,7 +2,7 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::uprops_helpers::{self, get_last_component_no_version, TomlBinary}; +use crate::uprops::uprops_helpers::{self, get_last_component_no_version, TomlBinary}; use icu_properties::provider::UnicodePropertyV1; use icu_properties::provider::UnicodePropertyV1Marker; diff --git a/provider/uprops/src/casemapping.rs b/provider/datagen/src/uprops/casemapping.rs similarity index 99% rename from provider/uprops/src/casemapping.rs rename to provider/datagen/src/uprops/casemapping.rs index 83f244c35c2..0d5b5ebc1de 100644 --- a/provider/uprops/src/casemapping.rs +++ b/provider/datagen/src/uprops/casemapping.rs @@ -2,7 +2,7 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::uprops_serde; +use crate::uprops::uprops_serde; use icu_casemapping::provider::{CaseMappingV1, CaseMappingV1Marker}; use icu_casemapping::CaseMappingInternals; use icu_codepointtrie::CodePointTrieHeader; @@ -64,7 +64,7 @@ impl ResourceProvider for CaseMappingDataProvider { #[cfg(test)] mod tests { - use crate::casemapping::CaseMappingDataProvider; + use crate::uprops::casemapping::CaseMappingDataProvider; use core::str::FromStr; use icu_casemapping::CaseMapping; use icu_locid::Locale; diff --git a/provider/uprops/src/enum_codepointtrie.rs b/provider/datagen/src/uprops/enum_codepointtrie.rs similarity index 98% rename from provider/uprops/src/enum_codepointtrie.rs rename to provider/datagen/src/uprops/enum_codepointtrie.rs index f39ad4f57d2..74872621680 100644 --- a/provider/uprops/src/enum_codepointtrie.rs +++ b/provider/datagen/src/uprops/enum_codepointtrie.rs @@ -2,8 +2,8 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::uprops_helpers::{self, get_last_component_no_version, TomlEnumerated}; -use crate::uprops_serde::SerializedCodePointTrie; +use crate::uprops::uprops_helpers::{self, get_last_component_no_version, TomlEnumerated}; +use crate::uprops::uprops_serde::SerializedCodePointTrie; use icu_codepointtrie::{CodePointTrie, CodePointTrieHeader, TrieType, TrieValue}; use icu_properties::provider::*; diff --git a/provider/uprops/src/enum_uniset.rs b/provider/datagen/src/uprops/enum_uniset.rs similarity index 97% rename from provider/uprops/src/enum_uniset.rs rename to provider/datagen/src/uprops/enum_uniset.rs index b94e9648088..b3e989b9fda 100644 --- a/provider/uprops/src/enum_uniset.rs +++ b/provider/datagen/src/uprops/enum_uniset.rs @@ -2,7 +2,7 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::uprops_helpers::{self, get_last_component_no_version, TomlEnumerated}; +use crate::uprops::uprops_helpers::{self, get_last_component_no_version, TomlEnumerated}; use icu_properties::provider::UnicodePropertyV1; use icu_properties::provider::UnicodePropertyV1Marker; diff --git a/provider/uprops/src/lib.rs b/provider/datagen/src/uprops/mod.rs similarity index 77% rename from provider/uprops/src/lib.rs rename to provider/datagen/src/uprops/mod.rs index ea8e6ec3ffb..4c61d52d3dd 100644 --- a/provider/uprops/src/lib.rs +++ b/provider/datagen/src/uprops/mod.rs @@ -2,31 +2,17 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -//! `icu_provider_uprops` contains implementations of the [`ICU4X`] +//! `icu_datagen::uprops` contains implementations of the [`ICU4X`] //! [data provider] interface backed by TOML files exported by the //! ICU4C icuwriteuprops tool. Create a directory containing TOML files for //! the necessary Unicode properties and then pass the path to the desired //! provider. //! //! **Important:** This data provider implementation is not optimized -//! for production use. It is much more efficient if you use -//! [`FsDataProvider`] or [`StaticDataProvider`] instead. +//! for production use. Read more in the [data provider] docs. //! //! [`ICU4X`]: ../icu/index.html //! [data provider]: icu_provider -//! [`FsDataProvider`]: ../icu_provider_fs/struct.FsDataProvider.html -//! [`StaticDataProvider`]: ../icu_provider_blob/struct.StaticDataProvider.html - -// https://github.com/unicode-org/icu4x/blob/main/docs/process/boilerplate.md#library-annotations -#![cfg_attr( - not(test), - deny( - clippy::indexing_slicing, - clippy::unwrap_used, - clippy::expect_used, - clippy::panic - ) -)] mod bin_uniset; mod enum_codepointtrie; diff --git a/provider/uprops/src/reader.rs b/provider/datagen/src/uprops/reader.rs similarity index 98% rename from provider/uprops/src/reader.rs rename to provider/datagen/src/uprops/reader.rs index 4d02726f5ed..eaa6a5e0e02 100644 --- a/provider/uprops/src/reader.rs +++ b/provider/datagen/src/uprops/reader.rs @@ -11,7 +11,6 @@ use std::path::{Path, PathBuf}; /// Helper function to open a file and return failures as a crate error. pub fn open_reader(path: &Path) -> eyre::Result> { - #[cfg(feature = "log")] log::trace!("Reading: {:?}", path); File::open(&path) .map(BufReader::new) diff --git a/provider/uprops/src/script.rs b/provider/datagen/src/uprops/script.rs similarity index 99% rename from provider/uprops/src/script.rs rename to provider/datagen/src/uprops/script.rs index 18d15e8c591..926b98c089f 100644 --- a/provider/uprops/src/script.rs +++ b/provider/datagen/src/uprops/script.rs @@ -2,8 +2,8 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::uprops_helpers; -use crate::uprops_serde::script_extensions::ScriptWithExtensionsProperty; +use crate::uprops::uprops_helpers; +use crate::uprops::uprops_serde::script_extensions::ScriptWithExtensionsProperty; use icu_codepointtrie::CodePointTrie; use icu_properties::provider::{ key, ScriptWithExtensionsPropertyV1, ScriptWithExtensionsPropertyV1Marker, diff --git a/provider/uprops/src/uprops_helpers.rs b/provider/datagen/src/uprops/uprops_helpers.rs similarity index 97% rename from provider/uprops/src/uprops_helpers.rs rename to provider/datagen/src/uprops/uprops_helpers.rs index f44a2119e3f..4885b280452 100644 --- a/provider/uprops/src/uprops_helpers.rs +++ b/provider/datagen/src/uprops/uprops_helpers.rs @@ -2,9 +2,9 @@ // called LICENSE at the top level of the ICU4X source tree // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). -use crate::reader::*; +use crate::uprops::reader::*; -use crate::uprops_serde; +use crate::uprops::uprops_serde; use eyre::{eyre, WrapErr}; use std::collections::HashMap; use std::path::Path; diff --git a/provider/uprops/src/uprops_serde.rs b/provider/datagen/src/uprops/uprops_serde.rs similarity index 100% rename from provider/uprops/src/uprops_serde.rs rename to provider/datagen/src/uprops/uprops_serde.rs diff --git a/provider/cldr/tests/testdata/dummy.pdf b/provider/datagen/tests/testdata/dummy.pdf similarity index 100% rename from provider/cldr/tests/testdata/dummy.pdf rename to provider/datagen/tests/testdata/dummy.pdf diff --git a/provider/cldr/tests/testdata/dummy.zip b/provider/datagen/tests/testdata/dummy.zip similarity index 100% rename from provider/cldr/tests/testdata/dummy.zip rename to provider/datagen/tests/testdata/dummy.zip diff --git a/tools/datagen/tests/testdata/work_log+keys.txt b/provider/datagen/tests/testdata/work_log+keys.txt similarity index 100% rename from tools/datagen/tests/testdata/work_log+keys.txt rename to provider/datagen/tests/testdata/work_log+keys.txt diff --git a/provider/segmenter/Cargo.toml b/provider/segmenter/Cargo.toml deleted file mode 100644 index 4b34524255b..00000000000 --- a/provider/segmenter/Cargo.toml +++ /dev/null @@ -1,39 +0,0 @@ -# This file is part of ICU4X. For terms of use, please see the file -# called LICENSE at the top level of the ICU4X source tree -# (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). - -[package] -name = "icu_provider_segmenter" -description = "Data provider that serialize and deserialize segmenter rule data" -version = "0.1.0" -authors = ["The ICU4X Project Developers"] -edition = "2021" -readme = "README.md" -repository = "https://github.com/unicode-org/icu4x" -license-file = "LICENSE" -categories = ["internationalization"] -# Keep this in sync with other crates unless there are exceptions -include = [ - "src/**/*", - "examples/**/*", - "benches/**/*", - "tests/**/*", - "Cargo.toml", - "LICENSE", - "README.md" -] - -[dependencies] -icu_codepointtrie = { version = "0.3.3", path = "../../utils/codepointtrie" } -icu_properties = { version = "0.5", path = "../../components/properties", features = ["datagen"] } -icu_provider = { version = "0.5", path = "../../provider/core", features = ["serialize", "macros"] } -icu_provider_adapters = { path = "../../provider/adapters" } -icu_provider_uprops = { version = "0.5", path = "../../provider/uprops" } -icu_segmenter = { version = "0.1", path = "../../experimental/segmenter", features = ["datagen"] } -icu_testdata = { version = "0.5", path = "../../provider/testdata" } -serde = { version = "1.0", features = ["derive"] } -toml = { version = "0.5" } -zerovec = { path = "../../utils/zerovec", features = ["serde", "yoke"] } - -[features] -default = [] diff --git a/provider/segmenter/LICENSE b/provider/segmenter/LICENSE deleted file mode 100644 index 5ab1f57507b..00000000000 --- a/provider/segmenter/LICENSE +++ /dev/null @@ -1,331 +0,0 @@ -Except as otherwise noted below, ICU4X is licensed under the Apache -License, Version 2.0 (included below) or the MIT license (included -below), at your option. Unless importing data or code in the manner -stated below, any contribution intentionally submitted for inclusion -in ICU4X by you, as defined in the Apache-2.0 license, shall be dual -licensed in the foregoing manner, without any additional terms or -conditions. - -As exceptions to the above: -* Portions of ICU4X that have been adapted from ICU4C and/or ICU4J are -under the Unicode license (included below) and/or the ICU license -(included below) as indicated by source code comments. -* Unicode data incorporated in ICU4X is under the Unicode license -(included below). -* Your contributions may import code from ICU4C and/or ICU4J and -Unicode data under these licenses. Indicate the license and the ICU4C -or ICU4J origin in source code comments. - -- - - - - -Apache License, version 2.0 - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -- - - - - -MIT License - -Copyright The ICU4X Authors - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -- - - - - -Unicode License - -COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later) - -Copyright © 1991-2020 Unicode, Inc. All rights reserved. -Distributed under the Terms of Use in https://www.unicode.org/copyright.html. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of the Unicode data files and any associated documentation -(the "Data Files") or Unicode software and any associated documentation -(the "Software") to deal in the Data Files or Software -without restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, and/or sell copies of -the Data Files or Software, and to permit persons to whom the Data Files -or Software are furnished to do so, provided that either -(a) this copyright and permission notice appear with all copies -of the Data Files or Software, or -(b) this copyright and permission notice appear in associated -Documentation. - -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT OF THIRD PARTY RIGHTS. -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS -NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL -DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, -DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THE DATA FILES OR SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, -use or other dealings in these Data Files or Software without prior -written authorization of the copyright holder. - -- - - - - -ICU License - ICU 1.8.1 to ICU 57.1 - -COPYRIGHT AND PERMISSION NOTICE - -Copyright (c) 1995-2016 International Business Machines Corporation and others -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, and/or sell copies of the Software, and to permit persons -to whom the Software is furnished to do so, provided that the above -copyright notice(s) and this permission notice appear in all copies of -the Software and that both the above copyright notice(s) and this -permission notice appear in supporting documentation. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY -SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER -RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, use -or other dealings in this Software without prior written authorization -of the copyright holder. - -All trademarks and registered trademarks mentioned herein are the -property of their respective owners. - -- - - - diff --git a/provider/segmenter/README.md b/provider/segmenter/README.md deleted file mode 100644 index 843a7d981d2..00000000000 --- a/provider/segmenter/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# icu_provider_segmenter [![crates.io](https://img.shields.io/crates/v/icu_provider_segmenter)](https://crates.io/crates/icu_provider_segmenter) - -`icu_provider_segmenter` contains implementations of the [`ICU4X`] [data provider] interface -based on Unicode properties and TOML files implementing [Unicode Standard Annex #14][UAX14] and -[Unicode Standard Annex #29][UAX29] breaking rules. - -**Important:** This provider implementation is not optimized for production use. It is much more -efficient if you use [`FsDataProvider`] or [`StaticDataProvider`] instead. - -## Examples - -```rust -use icu_provider_segmenter::SegmenterRuleProvider; -let provider = SegmenterRuleProvider::try_new( - "/path/to/segmenter/data/directory", - "/path/to/uprops/data/directory", -) -.expect_err("Specify a real directory in the line above"); -``` - -## Exporting data - -To generate the data required by the segmenters, run `cargo make testdata` from the top level. - -[`ICU4X`]: ../icu/index.html -[data provider]: icu_provider -[UAX14]: https://www.unicode.org/reports/tr14/ -[UAX29]: https://www.unicode.org/reports/tr29/ -[`FsDataProvider`]: ../icu_provider_fs/struct.FsDataProvider.html -[`StaticDataProvider`]: ../icu_provider_blob/struct.StaticDataProvider.html - -## More Information - -For more information on development, authorship, contributing etc. please visit [`ICU4X home page`](https://github.com/unicode-org/icu4x). diff --git a/provider/uprops/Cargo.toml b/provider/uprops/Cargo.toml deleted file mode 100644 index f3aee7d96ff..00000000000 --- a/provider/uprops/Cargo.toml +++ /dev/null @@ -1,50 +0,0 @@ -# This file is part of ICU4X. For terms of use, please see the file -# called LICENSE at the top level of the ICU4X source tree -# (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). - -[package] -name = "icu_provider_uprops" -description = "Data provider that reads from `icuwriteuprops` output from ICU4C" -version = "0.5.0" -authors = ["The ICU4X Project Developers"] -edition = "2018" -readme = "README.md" -repository = "https://github.com/unicode-org/icu4x" -license-file = "LICENSE" -categories = ["internationalization"] -# Keep this in sync with other crates unless there are exceptions -include = [ - "src/**/*", - "examples/**/*", - "benches/**/*", - "tests/**/*", - "Cargo.toml", - "LICENSE", - "README.md" -] - -[package.metadata.docs.rs] -all-features = true - -[dependencies] -displaydoc = { version = "0.2.3", default-features = false } -icu_casemapping = { version = "0.1", path = "../../experimental/casemapping", features = ["provider_serde", "provider_transform_internals"], optional = true } -icu_codepointtrie = { version = "0.3.3", path = "../../utils/codepointtrie", features = ["serialize"] } -icu_properties = { version = "0.5", path = "../../components/properties", features = ["datagen", "std"] } -icu_provider = { version = "0.5", path = "../../provider/core", features = ["serialize", "std", "datagen"] } -icu_provider_adapters = { path = "../../provider/adapters" } -icu_uniset = { version = "0.4.1", path = "../../utils/uniset", features = ["serialize"] } -serde = { version = "1.0", features = ["derive"] } -toml = { version = "0.5" } -zerovec = { version = "0.6", path = "../../utils/zerovec", features = ["serde", "yoke"] } -log = { version = "0.4", optional = true } -eyre = "0.6.5" -tinystr = "0.4" - -[dev-dependencies] -icu_locid = { version = "0.5", path = "../../components/locid" } -icu_testdata = { version = "0.5", path = "../../provider/testdata" } - -[features] -casemapping = ["icu_casemapping"] -default = ["casemapping"] diff --git a/provider/uprops/LICENSE b/provider/uprops/LICENSE deleted file mode 100644 index 5ab1f57507b..00000000000 --- a/provider/uprops/LICENSE +++ /dev/null @@ -1,331 +0,0 @@ -Except as otherwise noted below, ICU4X is licensed under the Apache -License, Version 2.0 (included below) or the MIT license (included -below), at your option. Unless importing data or code in the manner -stated below, any contribution intentionally submitted for inclusion -in ICU4X by you, as defined in the Apache-2.0 license, shall be dual -licensed in the foregoing manner, without any additional terms or -conditions. - -As exceptions to the above: -* Portions of ICU4X that have been adapted from ICU4C and/or ICU4J are -under the Unicode license (included below) and/or the ICU license -(included below) as indicated by source code comments. -* Unicode data incorporated in ICU4X is under the Unicode license -(included below). -* Your contributions may import code from ICU4C and/or ICU4J and -Unicode data under these licenses. Indicate the license and the ICU4C -or ICU4J origin in source code comments. - -- - - - - -Apache License, version 2.0 - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -- - - - - -MIT License - -Copyright The ICU4X Authors - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -- - - - - -Unicode License - -COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later) - -Copyright © 1991-2020 Unicode, Inc. All rights reserved. -Distributed under the Terms of Use in https://www.unicode.org/copyright.html. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of the Unicode data files and any associated documentation -(the "Data Files") or Unicode software and any associated documentation -(the "Software") to deal in the Data Files or Software -without restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, and/or sell copies of -the Data Files or Software, and to permit persons to whom the Data Files -or Software are furnished to do so, provided that either -(a) this copyright and permission notice appear with all copies -of the Data Files or Software, or -(b) this copyright and permission notice appear in associated -Documentation. - -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT OF THIRD PARTY RIGHTS. -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS -NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL -DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, -DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THE DATA FILES OR SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, -use or other dealings in these Data Files or Software without prior -written authorization of the copyright holder. - -- - - - - -ICU License - ICU 1.8.1 to ICU 57.1 - -COPYRIGHT AND PERMISSION NOTICE - -Copyright (c) 1995-2016 International Business Machines Corporation and others -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, and/or sell copies of the Software, and to permit persons -to whom the Software is furnished to do so, provided that the above -copyright notice(s) and this permission notice appear in all copies of -the Software and that both the above copyright notice(s) and this -permission notice appear in supporting documentation. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY -SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER -RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, use -or other dealings in this Software without prior written authorization -of the copyright holder. - -All trademarks and registered trademarks mentioned herein are the -property of their respective owners. - -- - - - diff --git a/provider/uprops/README.md b/provider/uprops/README.md deleted file mode 100644 index 4d279b5e8bf..00000000000 --- a/provider/uprops/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# icu_provider_uprops [![crates.io](https://img.shields.io/crates/v/icu_provider_uprops)](https://crates.io/crates/icu_provider_uprops) - -`icu_provider_uprops` contains implementations of the [`ICU4X`] -[data provider] interface backed by TOML files exported by the -ICU4C icuwriteuprops tool. Create a directory containing TOML files for -the necessary Unicode properties and then pass the path to the desired -provider. - -**Important:** This data provider implementation is not optimized -for production use. It is much more efficient if you use -[`FsDataProvider`] or [`StaticDataProvider`] instead. - -[`ICU4X`]: ../icu/index.html -[data provider]: icu_provider -[`FsDataProvider`]: ../icu_provider_fs/struct.FsDataProvider.html -[`StaticDataProvider`]: ../icu_provider_blob/struct.StaticDataProvider.html - -## More Information - -For more information on development, authorship, contributing etc. please visit [`ICU4X home page`](https://github.com/unicode-org/icu4x). diff --git a/tools/datagen/Cargo.toml b/tools/datagen/Cargo.toml deleted file mode 100644 index b0157965fa5..00000000000 --- a/tools/datagen/Cargo.toml +++ /dev/null @@ -1,67 +0,0 @@ -# This file is part of ICU4X. For terms of use, please see the file -# called LICENSE at the top level of the ICU4X source tree -# (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). - -[package] -name = "icu_datagen" -description = "Generate data for ICU4X DataProvider" -version = "0.5.0" -authors = ["The ICU4X Project Developers"] -edition = "2018" -readme = "README.md" -repository = "https://github.com/unicode-org/icu4x" -license-file = "LICENSE" -categories = ["internationalization"] -# Keep this in sync with other crates unless there are exceptions -include = [ - "src/**/*", - "examples/**/*", - "benches/**/*", - "tests/**/*", - "Cargo.toml", - "LICENSE", - "README.md", -] - -[package.metadata.docs.rs] -all-features = true - -[dependencies] -eyre = "0.6" -clap = "2.33" -futures = "0.3" -icu_locid = { version = "0.5", path = "../../components/locid", features = ["std"]} -icu_properties = { version = "0.5", path = "../../components/properties", features = ["std"]} -icu_provider = { version = "0.5", path = "../../provider/core", features = ["std", "log_error_context"]} -icu_provider_adapters = { path = "../../provider/adapters", features = ["datagen"] } -icu_provider_blob = { version = "0.5", path = "../../provider/blob", features = ["export"] } -icu_provider_cldr = { version = "0.5", path = "../../provider/cldr", features = ["download"] } -icu_provider_fs = { version = "0.5", path = "../../provider/fs", features = ["export"] } -icu_provider_uprops = { version = "0.5", path = "../../provider/uprops", features = ["log"] } -icu_provider_segmenter = { version = "0.1", path = "../../provider/segmenter" } -icu_segmenter = { version = "0.1", path = "../../experimental/segmenter" } -icu_testdata = { version = "0.5", path = "../../provider/testdata", features = ["metadata"] } -litemap = { version = "0.3.0", path = "../../utils/litemap" } -log = "0.4" -rayon = "1.5" -reqwest = { version = "0.11", features = ["json", "stream", "gzip"] } -serde = { version = "1.0", default-features = false, features = ["derive", "alloc"] } -simple_logger = "1.12" -tokio = { version = "1.5", features = ["rt-multi-thread", "macros", "fs"] } -writeable = { version = "0.3", path = "../../utils/writeable" } -dhat = "0.3.0" - -[features] -experimental = [] - -[[bin]] -name = "icu4x-datagen" -path = "src/bin/datagen.rs" - -[[bin]] -name = "icu4x-testdata-download" -path = "src/bin/testdata-download.rs" - -[[bin]] -name = "icu4x-verify-zero-copy" -path = "src/bin/verify-zero-copy.rs" diff --git a/tools/datagen/LICENSE b/tools/datagen/LICENSE deleted file mode 100644 index 5ab1f57507b..00000000000 --- a/tools/datagen/LICENSE +++ /dev/null @@ -1,331 +0,0 @@ -Except as otherwise noted below, ICU4X is licensed under the Apache -License, Version 2.0 (included below) or the MIT license (included -below), at your option. Unless importing data or code in the manner -stated below, any contribution intentionally submitted for inclusion -in ICU4X by you, as defined in the Apache-2.0 license, shall be dual -licensed in the foregoing manner, without any additional terms or -conditions. - -As exceptions to the above: -* Portions of ICU4X that have been adapted from ICU4C and/or ICU4J are -under the Unicode license (included below) and/or the ICU license -(included below) as indicated by source code comments. -* Unicode data incorporated in ICU4X is under the Unicode license -(included below). -* Your contributions may import code from ICU4C and/or ICU4J and -Unicode data under these licenses. Indicate the license and the ICU4C -or ICU4J origin in source code comments. - -- - - - - -Apache License, version 2.0 - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -- - - - - -MIT License - -Copyright The ICU4X Authors - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -- - - - - -Unicode License - -COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later) - -Copyright © 1991-2020 Unicode, Inc. All rights reserved. -Distributed under the Terms of Use in https://www.unicode.org/copyright.html. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of the Unicode data files and any associated documentation -(the "Data Files") or Unicode software and any associated documentation -(the "Software") to deal in the Data Files or Software -without restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, and/or sell copies of -the Data Files or Software, and to permit persons to whom the Data Files -or Software are furnished to do so, provided that either -(a) this copyright and permission notice appear with all copies -of the Data Files or Software, or -(b) this copyright and permission notice appear in associated -Documentation. - -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT OF THIRD PARTY RIGHTS. -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS -NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL -DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, -DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THE DATA FILES OR SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, -use or other dealings in these Data Files or Software without prior -written authorization of the copyright holder. - -- - - - - -ICU License - ICU 1.8.1 to ICU 57.1 - -COPYRIGHT AND PERMISSION NOTICE - -Copyright (c) 1995-2016 International Business Machines Corporation and others -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, and/or sell copies of the Software, and to permit persons -to whom the Software is furnished to do so, provided that the above -copyright notice(s) and this permission notice appear in all copies of -the Software and that both the above copyright notice(s) and this -permission notice appear in supporting documentation. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY -SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER -RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, use -or other dealings in this Software without prior written authorization -of the copyright holder. - -All trademarks and registered trademarks mentioned herein are the -property of their respective owners. - -- - - - diff --git a/tools/scripts/data.toml b/tools/scripts/data.toml index 8ead9fb25a4..78599c1c521 100644 --- a/tools/scripts/data.toml +++ b/tools/scripts/data.toml @@ -204,8 +204,8 @@ args = [ "--features=experimental", "--", "--input-from-testdata", - "--out=tools/datagen/tests/testdata/work_log_bincode", - "--key-file=tools/datagen/tests/testdata/work_log+keys.txt", + "--out=provider/datagen/tests/testdata/work_log_bincode", + "--key-file=provider/datagen/tests/testdata/work_log+keys.txt", "--locales=de", "--locales=es", "--syntax=bincode", diff --git a/tools/scripts/wasm.toml b/tools/scripts/wasm.toml index ad4a6afd4c2..c6b4ec9d150 100644 --- a/tools/scripts/wasm.toml +++ b/tools/scripts/wasm.toml @@ -372,7 +372,7 @@ script_runner = "@duckscript" script = ''' exit_on_error true -expected = readfile tools/datagen/tests/testdata/work_log+keys.txt +expected = readfile provider/datagen/tests/testdata/work_log+keys.txt actual = readfile wasmpkg/keys/work_log+keys.txt are_equal = eq ${expected} ${actual} @@ -383,7 +383,7 @@ else echo "work_log+keys.txt do not match! Actual generated output:" echo "" echo ${actual} - echo "If this is expected, copy the above output into tools/datagen/tests/testdata/work_log+keys.txt" + echo "If this is expected, copy the above output into provider/datagen/tests/testdata/work_log+keys.txt" echo "*****" exit 1 end