From b2231af211bdba0e7572ea4d6f19496b24962c2e Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 29 Nov 2023 16:32:25 +0100 Subject: [PATCH 1/3] remove lazy_static and use threa --- Cargo.lock | 190 ++++++++++++++++++++----- backends/candle/Cargo.toml | 4 +- backends/candle/src/compute_cap.rs | 50 +++++-- backends/candle/src/flash_attn.rs | 12 +- backends/candle/src/layers.rs | 2 +- backends/candle/src/layers/cublaslt.rs | 42 +++--- backends/candle/src/layers/linear.rs | 4 +- backends/candle/src/lib.rs | 97 ++++++++++++- backends/candle/src/models/bert.rs | 6 +- backends/candle/src/models/jina.rs | 6 +- backends/src/lib.rs | 80 ++++++----- core/src/infer.rs | 2 +- core/src/queue.rs | 2 +- core/src/tokenization.rs | 2 +- 14 files changed, 381 insertions(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 348db95e..f1548215 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -473,9 +473,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.8" +version = "4.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" +checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" dependencies = [ "clap_builder", "clap_derive", @@ -483,9 +483,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.8" +version = "4.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" +checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" dependencies = [ "anstream", "anstyle", @@ -673,7 +673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.2", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core", @@ -849,12 +849,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1241,9 +1241,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" @@ -1443,7 +1443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.2", + "hashbrown 0.14.3", "serde", ] @@ -1473,6 +1473,20 @@ dependencies = [ "tracing-opentelemetry 0.21.0", ] +[[package]] +name = "insta" +version = "1.34.0" +source = "git+https://github.com/OlivierDehaene/insta?rev=f4f98c0410b91fb5a28b10df98e4422955be9c2c#f4f98c0410b91fb5a28b10df98e4422955be9c2c" +dependencies = [ + "console", + "lazy_static", + "linked-hash-map", + "serde", + "serde_yaml", + "similar", + "yaml-rust", +] + [[package]] name = "instant" version = "0.1.12" @@ -1536,9 +1550,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" -version = "0.3.65" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" dependencies = [ "wasm-bindgen", ] @@ -1572,6 +1586,12 @@ dependencies = [ "redox_syscall 0.4.1", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.4.11" @@ -2311,9 +2331,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" dependencies = [ "unicode-ident", ] @@ -2674,9 +2694,9 @@ checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1" [[package]] name = "ring" -version = "0.17.5" +version = "0.17.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866" dependencies = [ "cc", "getrandom", @@ -2786,9 +2806,9 @@ dependencies = [ [[package]] name = "safetensors" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1e5186bd51ae3f90999d243853f5e8cb51f3467f55da42dc611ed2342483dad" +checksum = "1659ef1c27917eb58c2d53664b5506d0b68c9cb9b460d3e0901011cf71269a8e" dependencies = [ "serde", "serde_json", @@ -2910,6 +2930,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" +dependencies = [ + "indexmap 2.1.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha2" version = "0.10.8" @@ -2948,6 +2981,12 @@ dependencies = [ "libc", ] +[[package]] +name = "similar" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597" + [[package]] name = "sketches-ddsketch" version = "0.2.1" @@ -3140,10 +3179,10 @@ dependencies = [ "candle-layer-norm", "candle-nn", "candle-transformers", + "insta", "intel-mkl-src", - "lazy_static", "memmap2 0.9.0", - "safetensors 0.4.0", + "safetensors 0.4.1", "serde", "serde_json", "text-embeddings-backend-core", @@ -3779,6 +3818,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unsafe-libyaml" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" + [[package]] name = "untrusted" version = "0.9.0" @@ -3955,9 +4000,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3965,9 +4010,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" dependencies = [ "bumpalo", "log", @@ -3980,9 +4025,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" dependencies = [ "cfg-if", "js-sys", @@ -3992,9 +4037,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4002,9 +4047,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", @@ -4015,9 +4060,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" [[package]] name = "wav" @@ -4030,9 +4075,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.65" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" dependencies = [ "js-sys", "wasm-bindgen", @@ -4114,6 +4159,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -4144,6 +4198,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -4156,6 +4225,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -4168,6 +4243,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -4180,6 +4261,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -4192,6 +4279,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -4204,6 +4297,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -4216,6 +4315,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -4228,6 +4333,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winreg" version = "0.50.0" @@ -4247,6 +4358,15 @@ dependencies = [ "libc", ] +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "yoke" version = "0.7.3" diff --git a/backends/candle/Cargo.toml b/backends/candle/Cargo.toml index 429eb821..b83a50c2 100644 --- a/backends/candle/Cargo.toml +++ b/backends/candle/Cargo.toml @@ -15,7 +15,6 @@ candle-flash-attn = { version = "0.3.0", optional = true } candle-flash-attn-v1 = { git = "https://github.com/huggingface/candle-flash-attn-v1", rev = "62b75f1ea4e0961fad7b983ee8d723ed6fd68be5", optional = true } candle-cublaslt = { git = "https://github.com/huggingface/candle-cublaslt", rev = "58684e116aae248c353f87846ddf0b2a8a7ed855", optional = true } candle-layer-norm = { git = "https://github.com/huggingface/candle-layer-norm", rev = "5ed96012a693dff9685320765dd55a57fdaecdd6", optional = true } -lazy_static = "^1.4" text-embeddings-backend-core = { path = "../core" } tracing = "^0.1" safetensors = "^0.4" @@ -24,6 +23,9 @@ serde = { version = "^1.0", features = ["serde_derive"] } serde_json = "^1.0" memmap2 = "^0.9" +[dev-dependencies] +insta = { git = "https://github.com/OlivierDehaene/insta", rev = "f4f98c0410b91fb5a28b10df98e4422955be9c2c", features = ["yaml"] } + [build-dependencies] anyhow = { version = "1", features = ["backtrace"] } diff --git a/backends/candle/src/compute_cap.rs b/backends/candle/src/compute_cap.rs index b4bdf5fb..a1c391ee 100644 --- a/backends/candle/src/compute_cap.rs +++ b/backends/candle/src/compute_cap.rs @@ -2,20 +2,40 @@ use candle::cuda_backend::cudarc::driver::sys::CUdevice_attribute::{ CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, }; use candle::cuda_backend::cudarc::driver::CudaDevice; -use lazy_static::lazy_static; +use std::sync::Once; -lazy_static! { - pub static ref RUNTIME_COMPUTE_CAP: usize = { - let device = CudaDevice::new(0).expect("cuda is not available"); - let major = device - .attribute(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR) - .unwrap(); - let minor = device - .attribute(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR) - .unwrap(); - (major * 10 + minor) as usize - }; - pub static ref COMPILE_COMPUTE_CAP: usize = env!("CUDA_COMPUTE_CAP").parse::().unwrap(); +static INIT: Once = Once::new(); +static mut RUNTIME_COMPUTE_CAP: usize = 0; +static mut COMPILE_COMPUTE_CAP: usize = 0; + +fn init_compute_caps() { + unsafe { + INIT.call_once(|| { + let device = CudaDevice::new(0).expect("cuda is not available"); + let major = device + .attribute(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR) + .unwrap(); + let minor = device + .attribute(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR) + .unwrap(); + RUNTIME_COMPUTE_CAP = (major * 10 + minor) as usize; + COMPILE_COMPUTE_CAP = env!("CUDA_COMPUTE_CAP").parse::().unwrap(); + }); + } +} + +pub fn get_compile_compute_cap() -> usize { + unsafe { + init_compute_caps(); + COMPILE_COMPUTE_CAP + } +} + +pub fn get_runtime_compute_cap() -> usize { + unsafe { + init_compute_caps(); + RUNTIME_COMPUTE_CAP + } } fn compute_cap_matching(runtime_compute_cap: usize, compile_compute_cap: usize) -> bool { @@ -30,8 +50,8 @@ fn compute_cap_matching(runtime_compute_cap: usize, compile_compute_cap: usize) } pub fn incompatible_compute_cap() -> bool { - let compile_compute_cap = *COMPILE_COMPUTE_CAP; - let runtime_compute_cap = *RUNTIME_COMPUTE_CAP; + let compile_compute_cap = get_compile_compute_cap(); + let runtime_compute_cap = get_runtime_compute_cap(); !compute_cap_matching(runtime_compute_cap, compile_compute_cap) } diff --git a/backends/candle/src/flash_attn.rs b/backends/candle/src/flash_attn.rs index cf6189f0..23a752ae 100644 --- a/backends/candle/src/flash_attn.rs +++ b/backends/candle/src/flash_attn.rs @@ -1,4 +1,4 @@ -use crate::compute_cap::RUNTIME_COMPUTE_CAP; +use crate::compute_cap::get_runtime_compute_cap; use candle::Tensor; #[allow(clippy::too_many_arguments, unused)] @@ -13,7 +13,9 @@ pub(crate) fn flash_attn_varlen( softmax_scale: f32, causal: bool, ) -> Result { - if *RUNTIME_COMPUTE_CAP == 75 { + let runtime_compute_cap = get_runtime_compute_cap(); + + if runtime_compute_cap == 75 { #[cfg(feature = "flash-attn-v1")] { use candle_flash_attn_v1::flash_attn_varlen; @@ -31,7 +33,7 @@ pub(crate) fn flash_attn_varlen( } #[cfg(not(feature = "flash-attn-v1"))] candle::bail!("Flash attention v1 is not installed. Use `flash-attn-v1` feature.") - } else if (80..90).contains(&*RUNTIME_COMPUTE_CAP) { + } else if (80..90).contains(&runtime_compute_cap) { #[cfg(feature = "flash-attn")] { use candle_flash_attn::flash_attn_varlen; @@ -49,7 +51,7 @@ pub(crate) fn flash_attn_varlen( } #[cfg(not(feature = "flash-attn"))] candle::bail!("Flash attention is not installed. Use `flash-attn-v1` feature.") - } else if *RUNTIME_COMPUTE_CAP == 90 { + } else if runtime_compute_cap == 90 { #[cfg(feature = "flash-attn")] { use candle_flash_attn::flash_attn_varlen; @@ -70,6 +72,6 @@ pub(crate) fn flash_attn_varlen( } candle::bail!( "GPU with CUDA capability {} is not supported", - *RUNTIME_COMPUTE_CAP + runtime_compute_cap ); } diff --git a/backends/candle/src/layers.rs b/backends/candle/src/layers.rs index f0bf7e60..8e108fc2 100644 --- a/backends/candle/src/layers.rs +++ b/backends/candle/src/layers.rs @@ -3,6 +3,6 @@ mod cublaslt; mod layer_norm; mod linear; -pub use cublaslt::CUBLASLT; +pub use cublaslt::get_cublas_lt_wrapper; pub use layer_norm::LayerNorm; pub use linear::{HiddenAct, Linear}; diff --git a/backends/candle/src/layers/cublaslt.rs b/backends/candle/src/layers/cublaslt.rs index 08107c99..58685b45 100644 --- a/backends/candle/src/layers/cublaslt.rs +++ b/backends/candle/src/layers/cublaslt.rs @@ -1,28 +1,34 @@ use crate::layers::HiddenAct; use candle::{Device, Result, Tensor}; -use lazy_static::lazy_static; +use std::sync::Once; #[cfg(feature = "cuda")] use candle_cublaslt::{fused_batch_matmul, fused_matmul, Activation, CublasLt}; -lazy_static! { - pub static ref CUBLASLT: Option = { - match Device::cuda_if_available(0) { - Ok(device) => { - #[cfg(feature = "cuda")] - { - Some(CublasLtWrapper { - cublaslt: CublasLt::new(&device).unwrap(), - }) - } - #[cfg(not(feature = "cuda"))] - { - None +static INIT: Once = Once::new(); +static mut CUBLASLT: Option = None; + +pub fn get_cublas_lt_wrapper() -> Option<&'static CublasLtWrapper> { + unsafe { + INIT.call_once(|| { + CUBLASLT = match Device::cuda_if_available(0) { + Ok(device) => { + #[cfg(feature = "cuda")] + { + Some(CublasLtWrapper { + cublaslt: CublasLt::new(&device).unwrap(), + }) + } + #[cfg(not(feature = "cuda"))] + { + None + } } - } - Err(_) => None, - } - }; + Err(_) => None, + }; + }); + CUBLASLT.as_ref() + } } #[derive(Debug, Clone)] diff --git a/backends/candle/src/layers/linear.rs b/backends/candle/src/layers/linear.rs index 19ce2dc5..e145d3a5 100644 --- a/backends/candle/src/layers/linear.rs +++ b/backends/candle/src/layers/linear.rs @@ -1,4 +1,4 @@ -use crate::layers::CUBLASLT; +use crate::layers::cublaslt::get_cublas_lt_wrapper; use candle::{Device, Result, Tensor, D}; use serde::Deserialize; @@ -33,7 +33,7 @@ impl Linear { let _enter = self.span.enter(); #[allow(unused)] - if let (Device::Cuda(_), Some(cublaslt)) = (x.device(), &*CUBLASLT) { + if let (Device::Cuda(_), Some(cublaslt)) = (x.device(), get_cublas_lt_wrapper()) { // fused matmul requires x to be dims2 let mut final_shape = x.dims().to_vec(); final_shape.pop(); diff --git a/backends/candle/src/lib.rs b/backends/candle/src/lib.rs index 880a47d5..eaa3fa61 100644 --- a/backends/candle/src/lib.rs +++ b/backends/candle/src/lib.rs @@ -7,7 +7,7 @@ mod layers; mod models; #[cfg(feature = "cuda")] -use crate::compute_cap::{incompatible_compute_cap, COMPILE_COMPUTE_CAP, RUNTIME_COMPUTE_CAP}; +use crate::compute_cap::{get_runtime_compute_cap, incompatible_compute_cap, get_compile_compute_cap}; #[cfg(feature = "cuda")] use crate::models::FlashBertModel; use crate::models::{BertModel, JinaBertModel, Model, PositionEmbeddingType}; @@ -94,7 +94,7 @@ impl CandleBackend { #[cfg(feature = "cuda")] { if incompatible_compute_cap() { - return Err(BackendError::Start(format!("Runtime compute cap {} is not compatible with compile time compute cap {}", *RUNTIME_COMPUTE_CAP, *COMPILE_COMPUTE_CAP))); + return Err(BackendError::Start(format!("Runtime compute cap {} is not compatible with compile time compute cap {}", get_runtime_compute_cap(), get_compile_compute_cap()))); } if cfg!(any(feature = "flash-attn", feature = "flash-attn-v1")) @@ -156,3 +156,96 @@ impl WrapErr for Result { self.map_err(|e| BackendError::Inference(e.to_string())) } } + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::{Context, Result}; + use candle_nn::seq; + use hf_hub::api::sync::ApiBuilder; + use hf_hub::{Repo, RepoType}; + use text_embeddings_backend_core::Pool; + use tokenizers::pre_tokenizers::metaspace::PrependScheme; + use tokenizers::{PreTokenizerWrapper, Tokenizer}; + + fn download_artifacts(model_id: &'static str) -> Result { + let mut builder = ApiBuilder::new().with_progress(false); + + let api = builder.build().unwrap(); + let api_repo = api.repo(Repo::new(model_id.to_string(), RepoType::Model)); + + api_repo.get("config.json")?; + api_repo.get("tokenizer.json")?; + + let model_root = match api_repo.get("model.safetensors") { + Ok(p) => p, + Err(_) => { + let p = api_repo.get("pytorch_model.bin")?; + tracing::warn!("`model.safetensors` not found. Using `pytorch_model.bin` instead. Model loading will be significantly slower."); + p + } + } + .parent().unwrap() + .to_path_buf(); + Ok(model_root) + } + + fn load_tokenizer(model_root: &PathBuf) -> Result { + // Load tokenizer + let tokenizer_path = model_root.join("tokenizer.json"); + let mut tokenizer = Tokenizer::from_file(tokenizer_path).expect("tokenizer.json not found"); + // See https://github.com/huggingface/tokenizers/pull/1357 + if let Some(pre_tokenizer) = tokenizer.get_pre_tokenizer() { + if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer { + // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer` + let mut m = m.clone(); + m.set_prepend_scheme(PrependScheme::First); + tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Metaspace(m)); + } else if let PreTokenizerWrapper::Sequence(s) = pre_tokenizer { + // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer` + let mut s = s.clone(); + for pre_tokenizer in s.get_pre_tokenizers_mut() { + if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer { + m.set_prepend_scheme(PrependScheme::First); + } + } + tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Sequence(s)); + } + } + + tokenizer.with_padding(None); + Ok(tokenizer) + } + + fn round(v: &Vec, digits: i32) -> Vec { + let b = 10f32.powi(digits); + v.iter().map(|v| f32::round(v * b) / b).collect() + } + + #[test] + fn test_mini() -> Result<()> { + let model_root = download_artifacts("sentence-transformers/all-MiniLM-L6-v2")?; + + let tokenizer = load_tokenizer(&model_root)?; + let backend = CandleBackend::new( + model_root, + "float32".to_string(), + ModelType::Embedding(Pool::Mean), + )?; + + let encoding = tokenizer.encode("What is Deep Learning?", true).unwrap(); + let seq_len = encoding.len(); + + let batch = Batch { + input_ids: encoding.get_ids().to_vec(), + token_type_ids: encoding.get_type_ids().to_vec(), + position_ids: (0..seq_len as u32).collect(), + cumulative_seq_lengths: vec![0, seq_len as u32], + max_length: seq_len as u32, + }; + + let embeddings = backend.embed(batch)?; + insta::assert_yaml_snapshot!(round(&embeddings[0], 3)); + Ok(()) + } +} diff --git a/backends/candle/src/models/bert.rs b/backends/candle/src/models/bert.rs index dc817d39..32b898ef 100644 --- a/backends/candle/src/models/bert.rs +++ b/backends/candle/src/models/bert.rs @@ -1,4 +1,4 @@ -use crate::layers::{HiddenAct, LayerNorm, Linear, CUBLASLT}; +use crate::layers::{get_cublas_lt_wrapper, HiddenAct, LayerNorm, Linear}; use crate::models::Model; use candle::{DType, Device, IndexOp, Module, Result, Tensor, D}; use candle_nn::{Embedding, VarBuilder}; @@ -185,7 +185,9 @@ impl BertAttention { let value_layer = &qkv[2]; #[allow(unused_variables)] - let context_layer = if let (Device::Cuda(_), Some(cublaslt)) = (device, &*CUBLASLT) { + let context_layer = if let (Device::Cuda(_), Some(cublaslt)) = + (device, get_cublas_lt_wrapper()) + { #[cfg(feature = "cuda")] { // cuBLASLt batch matmul implementation requires inputs to be dims3 diff --git a/backends/candle/src/models/jina.rs b/backends/candle/src/models/jina.rs index 076f7643..8fb9305f 100644 --- a/backends/candle/src/models/jina.rs +++ b/backends/candle/src/models/jina.rs @@ -1,5 +1,5 @@ use crate::alibi::build_alibi_tensor; -use crate::layers::{HiddenAct, LayerNorm, Linear, CUBLASLT}; +use crate::layers::{get_cublas_lt_wrapper, HiddenAct, LayerNorm, Linear}; use crate::models::Model; use crate::models::{Config, PositionEmbeddingType}; use candle::{DType, Device, IndexOp, Module, Result, Tensor, D}; @@ -159,7 +159,9 @@ impl BertAttention { let value_layer = &qkv[2]; #[allow(unused_variables)] - let context_layer = if let (Device::Cuda(_), Some(cublaslt)) = (device, &*CUBLASLT) { + let context_layer = if let (Device::Cuda(_), Some(cublaslt)) = + (device, get_cublas_lt_wrapper()) + { #[cfg(feature = "cuda")] { // cuBLASLt batch matmul implementation requires inputs to be dims3 diff --git a/backends/src/lib.rs b/backends/src/lib.rs index 1a8d241f..7eca2be5 100644 --- a/backends/src/lib.rs +++ b/backends/src/lib.rs @@ -1,6 +1,8 @@ mod dtype; use std::path::PathBuf; +use std::sync::Arc; +use std::thread::JoinHandle; use std::time::{Duration, Instant}; use text_embeddings_backend_core::Backend as CoreBackend; use tokio::sync::{mpsc, oneshot, watch}; @@ -21,6 +23,7 @@ pub struct Backend { backend_sender: mpsc::UnboundedSender, /// Health status health_receiver: watch::Receiver, + _backend_thread: Arc, pub padded_model: bool, pub max_batch_size: Option, pub model_type: ModelType, @@ -47,14 +50,13 @@ impl Backend { let max_batch_size = backend.max_batch_size(); let (health_sender, health_receiver) = watch::channel(false); - - tokio::task::spawn_blocking(move || { - backend_blocking_task(backend, backend_receiver, health_sender) - }); + let _backend_thread = + Arc::new(BackendThread::new(backend, backend_receiver, health_sender)); Ok(Self { backend_sender, health_receiver, + _backend_thread, padded_model, max_batch_size, model_type, @@ -158,35 +160,49 @@ fn init_backend( Err(BackendError::NoBackend) } -fn backend_blocking_task( - backend: Box, - mut command_receiver: mpsc::UnboundedReceiver, - health_sender: watch::Sender, -) { - while let Some(cmd) = command_receiver.blocking_recv() { - let start = Instant::now(); - let mut healthy = false; - match cmd { - BackendCommand::Health(span, sender) => { - let _span = span.entered(); - let _ = sender.send(backend.health().map(|_| healthy = true)); - } - BackendCommand::Embed(batch, span, sender) => { - let _span = span.entered(); - let _ = sender.send(backend.embed(batch).map(|e| { - healthy = true; - (e, start.elapsed()) - })); - } - BackendCommand::Predict(batch, span, sender) => { - let _span = span.entered(); - let _ = sender.send(backend.predict(batch).map(|e| { - healthy = true; - (e, start.elapsed()) - })); +#[derive(Debug)] +struct BackendThread(Option>); + +impl BackendThread { + fn new( + backend: Box, + mut backend_receiver: mpsc::UnboundedReceiver, + health_sender: watch::Sender, + ) -> Self { + let handle = std::thread::spawn(move || { + while let Some(cmd) = backend_receiver.blocking_recv() { + let start = Instant::now(); + let mut healthy = false; + match cmd { + BackendCommand::Health(span, sender) => { + let _span = span.entered(); + let _ = sender.send(backend.health().map(|_| healthy = true)); + } + BackendCommand::Embed(batch, span, sender) => { + let _span = span.entered(); + let _ = sender.send(backend.embed(batch).map(|e| { + healthy = true; + (e, start.elapsed()) + })); + } + BackendCommand::Predict(batch, span, sender) => { + let _span = span.entered(); + let _ = sender.send(backend.predict(batch).map(|e| { + healthy = true; + (e, start.elapsed()) + })); + } + }; + let _ = health_sender.send(healthy); } - }; - let _ = health_sender.send(healthy); + }); + Self(Some(handle)) + } +} + +impl Drop for BackendThread { + fn drop(&mut self) { + self.0.take().unwrap().join().unwrap(); } } diff --git a/core/src/infer.rs b/core/src/infer.rs index 4232ea96..3c7802a1 100644 --- a/core/src/infer.rs +++ b/core/src/infer.rs @@ -323,7 +323,7 @@ async fn backend_task( }; // Handle sending responses in another thread to avoid starving the backend - tokio::task::spawn_blocking(move || match results { + std::thread::spawn(move || match results { Ok((embeddings, inference_duration)) => { batch.0.into_iter().zip(embeddings).for_each(|(m, e)| { let _ = m.response_tx.send(Ok(InferResponse { diff --git a/core/src/queue.rs b/core/src/queue.rs index 47968047..f5e602a4 100644 --- a/core/src/queue.rs +++ b/core/src/queue.rs @@ -49,7 +49,7 @@ impl Queue { let (queue_sender, queue_receiver) = mpsc::unbounded_channel(); // Launch background queue task - tokio::task::spawn_blocking(move || { + std::thread::spawn(move || { queue_blocking_task( padded_model, max_batch_tokens, diff --git a/core/src/tokenization.rs b/core/src/tokenization.rs index 7055f310..a8ecd3f4 100644 --- a/core/src/tokenization.rs +++ b/core/src/tokenization.rs @@ -32,7 +32,7 @@ impl Tokenization { senders.push(tokenizer_sender); // Spawn worker - tokio::task::spawn_blocking(move || { + std::thread::spawn(move || { tokenizer_worker( tokenizer_clone, max_input_length, From 19e50a296e3300bb50398ee9dc8c234d30c09e5b Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 30 Nov 2023 12:40:53 +0100 Subject: [PATCH 2/3] feat: add integration tests --- Cargo.lock | 40 + Makefile | 12 + backends/candle/Cargo.toml | 5 + backends/candle/src/layers/linear.rs | 38 +- backends/candle/src/lib.rs | 97 +- backends/candle/tests/common.rs | 154 ++ .../snapshots/test_bert__emotions_batch.snap | 89 + .../snapshots/test_bert__emotions_single.snap | 33 + .../snapshots/test_bert__mini_batch.snap | 1157 ++++++++++ .../snapshots/test_bert__mini_single.snap | 389 ++++ .../test_flash_bert__emotions_batch.snap | 89 + .../test_flash_bert__emotions_single.snap | 33 + .../test_flash_bert__mini_batch.snap | 1157 ++++++++++ .../test_flash_bert__mini_single.snap | 389 ++++ .../snapshots/test_jina__jina_batch.snap | 1541 +++++++++++++ .../snapshots/test_jina__jina_single.snap | 517 +++++ backends/candle/tests/test_bert.rs | 75 + backends/candle/tests/test_flash_bert.rs | 84 + backends/candle/tests/test_jina.rs | 42 + backends/src/lib.rs | 2 +- router/Cargo.toml | 5 + router/src/lib.rs | 322 ++- router/src/logging.rs | 65 + router/src/main.rs | 376 +--- router/src/prometheus.rs | 2 +- router/tests/common.rs | 72 + .../test_http_embed__embeddings_batch.snap | 1925 +++++++++++++++++ .../test_http_embed__embeddings_single.snap | 389 ++++ .../test_http_predict__predictions_batch.snap | 285 +++ ...test_http_predict__predictions_single.snap | 61 + .../snapshots/test_http_rerank__ranks.snap | 15 + router/tests/test_http_embed.rs | 53 + router/tests/test_http_predict.rs | 67 + router/tests/test_http_rerank.rs | 49 + 34 files changed, 9143 insertions(+), 486 deletions(-) create mode 100644 Makefile create mode 100644 backends/candle/tests/common.rs create mode 100644 backends/candle/tests/snapshots/test_bert__emotions_batch.snap create mode 100644 backends/candle/tests/snapshots/test_bert__emotions_single.snap create mode 100644 backends/candle/tests/snapshots/test_bert__mini_batch.snap create mode 100644 backends/candle/tests/snapshots/test_bert__mini_single.snap create mode 100644 backends/candle/tests/snapshots/test_flash_bert__emotions_batch.snap create mode 100644 backends/candle/tests/snapshots/test_flash_bert__emotions_single.snap create mode 100644 backends/candle/tests/snapshots/test_flash_bert__mini_batch.snap create mode 100644 backends/candle/tests/snapshots/test_flash_bert__mini_single.snap create mode 100644 backends/candle/tests/snapshots/test_jina__jina_batch.snap create mode 100644 backends/candle/tests/snapshots/test_jina__jina_single.snap create mode 100644 backends/candle/tests/test_bert.rs create mode 100644 backends/candle/tests/test_flash_bert.rs create mode 100644 backends/candle/tests/test_jina.rs create mode 100644 router/src/logging.rs create mode 100644 router/tests/common.rs create mode 100644 router/tests/snapshots/test_http_embed__embeddings_batch.snap create mode 100644 router/tests/snapshots/test_http_embed__embeddings_single.snap create mode 100644 router/tests/snapshots/test_http_predict__predictions_batch.snap create mode 100644 router/tests/snapshots/test_http_predict__predictions_single.snap create mode 100644 router/tests/snapshots/test_http_rerank__ranks.snap create mode 100644 router/tests/test_http_embed.rs create mode 100644 router/tests/test_http_predict.rs create mode 100644 router/tests/test_http_rerank.rs diff --git a/Cargo.lock b/Cargo.lock index f1548215..4b798d47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1524,6 +1524,15 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is_close" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4d49e01f14871e71a0dce6f09eb72308bca74a4adb1bab5a8c34ee26979a8a2" +dependencies = [ + "num-traits", +] + [[package]] name = "itertools" version = "0.10.5" @@ -2943,6 +2952,31 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "serial_test" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" +dependencies = [ + "dashmap", + "futures", + "lazy_static", + "log", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "sha2" version = "0.10.8" @@ -3179,14 +3213,18 @@ dependencies = [ "candle-layer-norm", "candle-nn", "candle-transformers", + "hf-hub", "insta", "intel-mkl-src", + "is_close", "memmap2 0.9.0", "safetensors 0.4.1", "serde", "serde_json", + "serial_test", "text-embeddings-backend-core", "thiserror", + "tokenizers", "tracing", ] @@ -3237,6 +3275,8 @@ dependencies = [ "hf-hub", "http 0.2.11", "init-tracing-opentelemetry", + "insta", + "is_close", "metrics", "metrics-exporter-prometheus", "num_cpus", diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..71bcfb31 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +integration-tests: + cargo test --release + +cuda-integration-tests: + cargo test -F text-embeddings-backend-candle/cuda -F text-embeddings-backend-candle/flash-attn -F text-embeddings-router/candle-cuda --release + +integration-tests-review: + cargo insta test --review --release + +cuda-integration-tests-review: + cargo insta test --review --features "text-embeddings-backend-candle/cuda text-embeddings-backend-candle/flash-attn text-embeddings-router/candle-cuda" --release + diff --git a/backends/candle/Cargo.toml b/backends/candle/Cargo.toml index b83a50c2..537c0b28 100644 --- a/backends/candle/Cargo.toml +++ b/backends/candle/Cargo.toml @@ -25,6 +25,11 @@ memmap2 = "^0.9" [dev-dependencies] insta = { git = "https://github.com/OlivierDehaene/insta", rev = "f4f98c0410b91fb5a28b10df98e4422955be9c2c", features = ["yaml"] } +is_close = "0.1.3" +hf-hub = "0.3.2" +anyhow = "1.0.75" +tokenizers = { version = "^0.15.0", default-features = false, features = ["onig", "esaxx_fast"] } +serial_test = "2.0.0" [build-dependencies] anyhow = { version = "1", features = ["backtrace"] } diff --git a/backends/candle/src/layers/linear.rs b/backends/candle/src/layers/linear.rs index e145d3a5..afd9017f 100644 --- a/backends/candle/src/layers/linear.rs +++ b/backends/candle/src/layers/linear.rs @@ -1,5 +1,5 @@ use crate::layers::cublaslt::get_cublas_lt_wrapper; -use candle::{Device, Result, Tensor, D}; +use candle::{Device, Result, Tensor}; use serde::Deserialize; #[derive(Debug, Deserialize, PartialEq, Clone)] @@ -34,22 +34,26 @@ impl Linear { #[allow(unused)] if let (Device::Cuda(_), Some(cublaslt)) = (x.device(), get_cublas_lt_wrapper()) { - // fused matmul requires x to be dims2 - let mut final_shape = x.dims().to_vec(); - final_shape.pop(); - final_shape.push(self.weight.dims()[0]); - - let x = x.flatten_to(D::Minus2)?; - let result = cublaslt.matmul( - &self.weight, - &x, - None, - None, - None, - self.bias.as_ref(), - self.act.clone(), - )?; - result.reshape(final_shape) + match x.dims() { + &[bsize, _, _] => cublaslt.batch_matmul( + &self.weight.broadcast_left(bsize)?, + x, + None, + None, + None, + self.bias.as_ref(), + self.act.clone(), + ), + _ => cublaslt.matmul( + &self.weight, + x, + None, + None, + None, + self.bias.as_ref(), + self.act.clone(), + ), + } } else { let w = match x.dims() { &[bsize, _, _] => self.weight.broadcast_left(bsize)?.t()?, diff --git a/backends/candle/src/lib.rs b/backends/candle/src/lib.rs index eaa3fa61..2e2aea3c 100644 --- a/backends/candle/src/lib.rs +++ b/backends/candle/src/lib.rs @@ -7,7 +7,9 @@ mod layers; mod models; #[cfg(feature = "cuda")] -use crate::compute_cap::{get_runtime_compute_cap, incompatible_compute_cap, get_compile_compute_cap}; +use crate::compute_cap::{ + get_compile_compute_cap, get_runtime_compute_cap, incompatible_compute_cap, +}; #[cfg(feature = "cuda")] use crate::models::FlashBertModel; use crate::models::{BertModel, JinaBertModel, Model, PositionEmbeddingType}; @@ -156,96 +158,3 @@ impl WrapErr for Result { self.map_err(|e| BackendError::Inference(e.to_string())) } } - -#[cfg(test)] -mod tests { - use super::*; - use anyhow::{Context, Result}; - use candle_nn::seq; - use hf_hub::api::sync::ApiBuilder; - use hf_hub::{Repo, RepoType}; - use text_embeddings_backend_core::Pool; - use tokenizers::pre_tokenizers::metaspace::PrependScheme; - use tokenizers::{PreTokenizerWrapper, Tokenizer}; - - fn download_artifacts(model_id: &'static str) -> Result { - let mut builder = ApiBuilder::new().with_progress(false); - - let api = builder.build().unwrap(); - let api_repo = api.repo(Repo::new(model_id.to_string(), RepoType::Model)); - - api_repo.get("config.json")?; - api_repo.get("tokenizer.json")?; - - let model_root = match api_repo.get("model.safetensors") { - Ok(p) => p, - Err(_) => { - let p = api_repo.get("pytorch_model.bin")?; - tracing::warn!("`model.safetensors` not found. Using `pytorch_model.bin` instead. Model loading will be significantly slower."); - p - } - } - .parent().unwrap() - .to_path_buf(); - Ok(model_root) - } - - fn load_tokenizer(model_root: &PathBuf) -> Result { - // Load tokenizer - let tokenizer_path = model_root.join("tokenizer.json"); - let mut tokenizer = Tokenizer::from_file(tokenizer_path).expect("tokenizer.json not found"); - // See https://github.com/huggingface/tokenizers/pull/1357 - if let Some(pre_tokenizer) = tokenizer.get_pre_tokenizer() { - if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer { - // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer` - let mut m = m.clone(); - m.set_prepend_scheme(PrependScheme::First); - tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Metaspace(m)); - } else if let PreTokenizerWrapper::Sequence(s) = pre_tokenizer { - // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer` - let mut s = s.clone(); - for pre_tokenizer in s.get_pre_tokenizers_mut() { - if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer { - m.set_prepend_scheme(PrependScheme::First); - } - } - tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Sequence(s)); - } - } - - tokenizer.with_padding(None); - Ok(tokenizer) - } - - fn round(v: &Vec, digits: i32) -> Vec { - let b = 10f32.powi(digits); - v.iter().map(|v| f32::round(v * b) / b).collect() - } - - #[test] - fn test_mini() -> Result<()> { - let model_root = download_artifacts("sentence-transformers/all-MiniLM-L6-v2")?; - - let tokenizer = load_tokenizer(&model_root)?; - let backend = CandleBackend::new( - model_root, - "float32".to_string(), - ModelType::Embedding(Pool::Mean), - )?; - - let encoding = tokenizer.encode("What is Deep Learning?", true).unwrap(); - let seq_len = encoding.len(); - - let batch = Batch { - input_ids: encoding.get_ids().to_vec(), - token_type_ids: encoding.get_type_ids().to_vec(), - position_ids: (0..seq_len as u32).collect(), - cumulative_seq_lengths: vec![0, seq_len as u32], - max_length: seq_len as u32, - }; - - let embeddings = backend.embed(batch)?; - insta::assert_yaml_snapshot!(round(&embeddings[0], 3)); - Ok(()) - } -} diff --git a/backends/candle/tests/common.rs b/backends/candle/tests/common.rs new file mode 100644 index 00000000..653596ad --- /dev/null +++ b/backends/candle/tests/common.rs @@ -0,0 +1,154 @@ +use anyhow::Result; +use hf_hub::api::sync::ApiBuilder; +use hf_hub::{Repo, RepoType}; +use insta::internals::YamlMatcher; +use serde::{Deserialize, Serialize}; +use std::cmp::max; +use std::ops::Deref; +use std::path::PathBuf; +use text_embeddings_backend_core::Batch; +use tokenizers::pre_tokenizers::metaspace::PrependScheme; +use tokenizers::pre_tokenizers::sequence::Sequence; +use tokenizers::{Encoding, PreTokenizerWrapper, Tokenizer}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct Score(f32); + +impl Score { + fn is_close(&self, other: &Self, abs_tol: f32) -> bool { + is_close::default() + .abs_tol(abs_tol) + .is_close(self.0, other.0) + } +} + +impl PartialEq for Score { + fn eq(&self, other: &Self) -> bool { + // Default tolerance for equality + self.is_close(other, 4e-3) + } +} + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +pub struct SnapshotScores(Vec>); + +impl Deref for SnapshotScores { + type Target = Vec>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From>> for SnapshotScores { + fn from(value: Vec>) -> Self { + Self( + value + .into_iter() + .map(|v| v.into_iter().map(|s| Score(s)).collect()) + .collect(), + ) + } +} + +pub fn download_artifacts(model_id: &'static str) -> Result { + let builder = ApiBuilder::new().with_progress(false); + + let api = builder.build().unwrap(); + let api_repo = api.repo(Repo::new(model_id.to_string(), RepoType::Model)); + + api_repo.get("config.json")?; + api_repo.get("tokenizer.json")?; + + let model_root = match api_repo.get("model.safetensors") { + Ok(p) => p, + Err(_) => { + let p = api_repo.get("pytorch_model.bin")?; + tracing::warn!("`model.safetensors` not found. Using `pytorch_model.bin` instead. Model loading will be significantly slower."); + p + } + } + .parent().unwrap() + .to_path_buf(); + Ok(model_root) +} + +pub fn relative_matcher() -> YamlMatcher { + YamlMatcher::new() +} + +pub fn load_tokenizer(model_root: &PathBuf) -> Result { + // Load tokenizer + let tokenizer_path = model_root.join("tokenizer.json"); + let mut tokenizer = Tokenizer::from_file(tokenizer_path).expect("tokenizer.json not found"); + // See https://github.com/huggingface/tokenizers/pull/1357 + if let Some(pre_tokenizer) = tokenizer.get_pre_tokenizer() { + if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer { + // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer` + let mut m = m.clone(); + m.set_prepend_scheme(PrependScheme::First); + tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Metaspace(m)); + } else if let PreTokenizerWrapper::Sequence(s) = pre_tokenizer { + let pre_tokenizers = s.get_pre_tokenizers(); + // Check if we have a Metaspace pre tokenizer in the sequence + let has_metaspace = pre_tokenizers + .iter() + .find(|t| matches!(t, PreTokenizerWrapper::Metaspace(_))) + .is_some(); + + if has_metaspace { + let mut new_pre_tokenizers = Vec::with_capacity(s.get_pre_tokenizers().len()); + + for pre_tokenizer in pre_tokenizers { + if let PreTokenizerWrapper::WhitespaceSplit(_) = pre_tokenizer { + // Remove WhitespaceSplit + // This will be done by the Metaspace pre tokenizer + continue; + } + + let mut pre_tokenizer = pre_tokenizer.clone(); + + if let PreTokenizerWrapper::Metaspace(ref mut m) = pre_tokenizer { + m.set_prepend_scheme(PrependScheme::First); + } + new_pre_tokenizers.push(pre_tokenizer); + } + tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Sequence(Sequence::new( + new_pre_tokenizers, + ))); + } + } + } + + tokenizer.with_padding(None); + Ok(tokenizer) +} + +pub fn batch(encodings: Vec) -> Batch { + let mut input_ids = Vec::new(); + let mut token_type_ids = Vec::new(); + let mut position_ids = Vec::new(); + let mut cumulative_seq_lengths = Vec::with_capacity(encodings.len() + 1); + cumulative_seq_lengths.push(0); + + let mut max_length = 0; + let mut cumulative_length = 0; + + for encoding in encodings { + let encoding_length = encoding.len() as u32; + input_ids.extend(encoding.get_ids().to_vec()); + token_type_ids.extend(encoding.get_type_ids().to_vec()); + position_ids.extend(0..encoding_length); + cumulative_length += encoding_length; + cumulative_seq_lengths.push(cumulative_length); + max_length = max(max_length, encoding_length); + } + + Batch { + input_ids, + token_type_ids, + position_ids, + cumulative_seq_lengths, + max_length, + } +} diff --git a/backends/candle/tests/snapshots/test_bert__emotions_batch.snap b/backends/candle/tests/snapshots/test_bert__emotions_batch.snap new file mode 100644 index 00000000..3ef01c5c --- /dev/null +++ b/backends/candle/tests/snapshots/test_bert__emotions_batch.snap @@ -0,0 +1,89 @@ +--- +source: backends/candle/tests/test_bert.rs +expression: predictions_batch +--- +- - -6.548559 + - -6.302024 + - -4.8671727 + - -3.9600255 + - -4.6329865 + - -6.2816987 + - -6.069644 + - -5.7742686 + - -6.9259467 + - -6.1909447 + - -5.67395 + - -6.1698227 + - -7.513461 + - -6.865867 + - -7.186479 + - -7.128109 + - -8.210709 + - -7.0171394 + - -7.1321163 + - -8.533409 + - -6.2294865 + - -8.742306 + - -5.7792044 + - -8.657227 + - -8.258305 + - -6.64832 + - -7.4060283 + - 3.046496 +- - -5.8167515 + - -6.6119466 + - -5.2771955 + - -2.6306503 + - -4.6419163 + - -5.579778 + - -5.797174 + - -6.0305815 + - -5.8720746 + - 0.45377323 + - -3.0235887 + - -5.3944407 + - -5.186683 + - -6.2649117 + - -6.1962767 + - -6.97937 + - -5.5674877 + - -5.521044 + - -5.8899207 + - -4.8699703 + - -5.6259933 + - -7.6109924 + - -4.3881936 + - -6.039008 + - -4.934696 + - -0.6715916 + - -6.399376 + - -2.4499295 +- - -6.548559 + - -6.302024 + - -4.8671727 + - -3.9600255 + - -4.6329865 + - -6.2816987 + - -6.069644 + - -5.7742686 + - -6.9259467 + - -6.1909447 + - -5.67395 + - -6.1698227 + - -7.513461 + - -6.865867 + - -7.186479 + - -7.128109 + - -8.210709 + - -7.0171394 + - -7.1321163 + - -8.533409 + - -6.2294865 + - -8.742306 + - -5.7792044 + - -8.657227 + - -8.258305 + - -6.64832 + - -7.4060283 + - 3.046496 + diff --git a/backends/candle/tests/snapshots/test_bert__emotions_single.snap b/backends/candle/tests/snapshots/test_bert__emotions_single.snap new file mode 100644 index 00000000..dde6ccbb --- /dev/null +++ b/backends/candle/tests/snapshots/test_bert__emotions_single.snap @@ -0,0 +1,33 @@ +--- +source: backends/candle/tests/test_bert.rs +expression: predictions_single +--- +- - -6.548218 + - -6.3022184 + - -4.866901 + - -3.9598548 + - -4.6330333 + - -6.281371 + - -6.070333 + - -5.7753787 + - -6.9252844 + - -6.1905437 + - -5.674121 + - -6.169409 + - -7.5133595 + - -6.8658547 + - -7.185884 + - -7.1283603 + - -8.210392 + - -7.016874 + - -7.1315 + - -8.53309 + - -6.229343 + - -8.741868 + - -5.7791805 + - -8.657056 + - -8.258206 + - -6.6477957 + - -7.406438 + - 3.0466576 + diff --git a/backends/candle/tests/snapshots/test_bert__mini_batch.snap b/backends/candle/tests/snapshots/test_bert__mini_batch.snap new file mode 100644 index 00000000..3ad76bc1 --- /dev/null +++ b/backends/candle/tests/snapshots/test_bert__mini_batch.snap @@ -0,0 +1,1157 @@ +--- +source: backends/candle/tests/test_bert.rs +expression: embeddings_batch +--- +- - -0.5904484 + - -0.25218195 + - 0.14959621 + - 0.05505526 + - -0.11826475 + - -0.16704129 + - -0.23318417 + - -0.0704903 + - -0.19961402 + - -0.2959458 + - -0.7729306 + - 0.14422165 + - -0.29086623 + - -0.18537273 + - -0.9093197 + - -0.019904744 + - 0.02856647 + - 0.34820673 + - -1.0033797 + - -0.41697913 + - 0.19350834 + - 0.40060228 + - -0.39103433 + - -0.18893544 + - 0.45299938 + - 0.25677818 + - 0.17712198 + - -0.3142255 + - 0.12515345 + - -0.08413245 + - 0.65004975 + - -0.23057863 + - 0.18415941 + - 0.4963523 + - -0.36540174 + - 0.83785826 + - -0.5513201 + - 0.103302784 + - 0.0940645 + - -0.21010701 + - -0.29487512 + - -0.18855686 + - -0.009985788 + - 0.18686627 + - 0.9258731 + - 0.33936834 + - -0.08204167 + - -0.60297924 + - 0.1529275 + - 0.2101926 + - -0.20914623 + - -0.13881606 + - -0.38299662 + - 0.63747364 + - 0.003919348 + - 0.098739825 + - 0.27915862 + - -0.021564007 + - -0.5371316 + - 0.32010522 + - 0.4855331 + - -0.69396293 + - 0.024767732 + - 0.23580678 + - 0.29508504 + - -0.27021644 + - 0.048729945 + - 0.110520415 + - 0.5391848 + - -0.771077 + - 0.42122838 + - 0.47483942 + - -0.052862015 + - 0.07978022 + - 0.14361086 + - -0.43833405 + - 0.21757023 + - 0.055354383 + - 0.74647045 + - -0.21855037 + - 0.7267293 + - 0.54959774 + - -0.11634091 + - 0.36827073 + - 1.0121477 + - -0.5172255 + - 0.117366515 + - 0.0698515 + - -0.67828286 + - -0.5236744 + - -0.27104068 + - -0.5252708 + - -0.59517235 + - -0.0011284138 + - -0.29228482 + - -0.09563696 + - -0.05197223 + - -0.9138813 + - -0.4042197 + - 0.38407257 + - -0.022372637 + - 0.06314764 + - 0.41756183 + - -0.04670214 + - -0.069010004 + - 0.090586014 + - 0.42655104 + - 0.10947996 + - 0.31035274 + - -0.62466854 + - -0.19219613 + - 0.29427573 + - 0.18328457 + - -0.18981433 + - 0.16418755 + - -0.19344743 + - -0.10209768 + - -0.04093083 + - -0.14816917 + - 0.69175416 + - -0.8734301 + - -0.05981831 + - -0.24518713 + - -0.12859611 + - -0.2916654 + - -0.18874489 + - -0.3669198 + - -0.000000000000000000000000000000029240375 + - -0.003938394 + - -0.36723635 + - -0.4536573 + - 0.049564976 + - 0.3200263 + - -0.4268046 + - -0.20275964 + - 0.093998395 + - -0.42062044 + - 0.44631353 + - -0.14178619 + - 0.0815359 + - -0.13849926 + - 1.1363466 + - 0.3480273 + - 0.09552129 + - -0.64421636 + - 0.5022697 + - 0.83456355 + - -0.5952269 + - -0.06076554 + - 0.31910786 + - -0.17580135 + - 0.022548726 + - -0.054858353 + - -0.10727251 + - 0.1562703 + - 0.16639505 + - 0.21630454 + - -0.028444165 + - -0.5627595 + - 0.07629048 + - -0.16489701 + - -0.069293454 + - 0.3651604 + - 0.042008024 + - 0.018344846 + - -0.0626462 + - 0.3005414 + - -0.2635724 + - 0.34574392 + - 0.062163908 + - -0.061937597 + - -0.43312663 + - -0.2066199 + - -0.1683005 + - 0.20767058 + - -0.71167225 + - -0.6564812 + - -0.38578632 + - -0.052831333 + - -0.3264928 + - -0.18351099 + - -0.4606121 + - 0.2933788 + - 0.2779899 + - 0.21287741 + - 0.206459 + - 0.12392608 + - 0.18230744 + - 0.53405255 + - 0.39990216 + - -0.46907288 + - 0.66455895 + - -0.09669055 + - 0.16171496 + - 0.6648889 + - 0.68079907 + - 1.0480419 + - -0.094723225 + - 0.02977555 + - 0.09691883 + - 0.20310333 + - -0.57307446 + - -0.07695443 + - 0.08091426 + - 0.15485184 + - -0.26107487 + - -0.25665718 + - 0.78575295 + - -0.47057983 + - 0.3093873 + - -0.25569737 + - 0.0062563634 + - -0.5304047 + - 0.60793275 + - -0.24611141 + - -0.3020915 + - 0.26118788 + - 0.01478196 + - -0.70216304 + - -0.049419917 + - -0.14812419 + - -0.07927934 + - -0.16526112 + - 0.000000000000000000000000000000021845427 + - -0.25161034 + - 0.4341398 + - -0.87574375 + - 0.71753454 + - 0.2905553 + - 0.007794529 + - -0.6243053 + - 0.35534343 + - -0.5528342 + - 0.18967059 + - 0.20288529 + - -0.035637587 + - 0.45268387 + - 0.109326735 + - 0.1914785 + - -0.16920124 + - -0.3563299 + - -0.34934732 + - -0.47009516 + - -0.26941064 + - 0.2352554 + - 0.5886043 + - -0.3380012 + - -0.24570635 + - -0.15959099 + - -0.22487848 + - -0.25580263 + - 0.12511626 + - -0.10618205 + - 0.2850696 + - 0.17198822 + - -0.32835695 + - -0.67826843 + - 0.44815382 + - 0.2926607 + - 0.50614864 + - 0.59291375 + - -0.28529674 + - -0.2626424 + - -0.42411417 + - 0.21714236 + - -0.009686342 + - 0.18806958 + - 0.2543909 + - -0.31932625 + - -0.058950584 + - 0.0066510993 + - -0.043817382 + - 0.32712084 + - -0.19686058 + - 0.31065568 + - 0.027757645 + - 0.034487057 + - -0.35713312 + - -0.25857708 + - -0.07837208 + - -0.10901906 + - 0.18711178 + - 0.14383718 + - 0.43526158 + - -0.18126877 + - -0.4950599 + - -0.116322875 + - -0.18074515 + - -0.16935274 + - 0.805221 + - -0.59641445 + - 0.62855345 + - 0.032347683 + - 0.14520326 + - 0.809142 + - 0.108584665 + - 0.34700632 + - 0.580749 + - -0.53957874 + - -0.5434822 + - 0.05687144 + - -0.09498585 + - -0.33424398 + - -0.18342555 + - 0.39793792 + - -0.8092169 + - -0.10401372 + - 0.65306324 + - -0.09104957 + - 0.10358218 + - -0.24646153 + - 0.5131639 + - -0.35961035 + - -0.026082557 + - -0.24173585 + - 0.10468064 + - -0.63350576 + - 0.0022953025 + - -0.52731735 + - -0.000000091965795 + - -0.19923782 + - 0.08858619 + - 0.5564927 + - -0.80046386 + - 0.30863148 + - -0.18751955 + - 0.10183903 + - 0.82070285 + - 0.024335938 + - 0.07749426 + - 0.60264343 + - 0.014298631 + - -0.5296712 + - -0.15352051 + - 0.026842814 + - 0.61098564 + - 0.459986 + - -0.00048416216 + - 0.4151469 + - 0.024833662 + - 0.8714803 + - 0.3206982 + - 0.12014544 + - -0.055944566 + - 0.22077613 + - -0.6302576 + - 0.36912346 + - 0.680718 + - 0.106342845 + - -0.4253362 + - -0.20274441 + - 0.5338388 + - -0.1241984 + - -0.1653124 + - 0.019658208 + - 0.88533986 + - 0.3077552 + - -0.28941244 + - -0.058864314 + - -0.40539756 + - -0.4040249 + - 1.13756 + - 0.2780994 + - -0.06830321 + - -0.46814448 + - 0.089857385 + - 0.31686476 + - -0.054425247 + - 0.14721735 + - 0.11701803 + - -0.16131987 + - 0.5147092 + - 0.6204316 + - 0.64212894 + - -0.036693208 + - 0.1190122 + - 0.017730946 + - -0.5259076 + - -0.16212647 + - 0.7366949 + - -0.07890918 + - 0.54164237 + - 0.28229737 + - 0.27705735 +- - -0.11974101 + - -0.5844457 + - 0.34629655 + - -0.24922174 + - -0.33206654 + - -0.20476547 + - -0.14636044 + - -0.21495423 + - 0.091791846 + - -0.24933603 + - -0.2792046 + - 0.24733767 + - -0.29774117 + - 0.3146898 + - -0.52879363 + - 0.0061102286 + - 0.1900632 + - 0.16750965 + - -0.82518905 + - -0.26580805 + - -0.17501947 + - 0.22086483 + - -0.16438791 + - -0.37062547 + - 0.57108366 + - 0.32025096 + - 0.29808035 + - -0.27603158 + - 0.15451494 + - -0.40962404 + - 0.29551056 + - 0.15903194 + - -0.18612969 + - 0.17952164 + - -0.14390941 + - 0.40272313 + - -0.20527533 + - 0.13613865 + - -0.020470936 + - 0.00003437698 + - -0.16705933 + - 0.07439775 + - -0.06595416 + - 0.0060240105 + - 0.4535567 + - 0.20025726 + - -0.0932296 + - -0.34127095 + - -0.029177912 + - 0.039519396 + - -0.41620174 + - -0.088601574 + - -0.13821039 + - 0.09530662 + - -0.032441348 + - -0.021458125 + - 0.53757215 + - -0.27482384 + - -0.24947084 + - 0.38965786 + - -0.033831786 + - -0.20488667 + - 0.004584587 + - 0.5945821 + - 0.20198834 + - 0.34131834 + - 0.18101013 + - 0.14449562 + - 0.12153621 + - 0.03891386 + - 0.43715325 + - 0.44023752 + - -0.00033312663 + - 0.6332288 + - 0.1355594 + - 0.35401857 + - 0.48780644 + - 0.2558096 + - 0.50310004 + - 0.032830365 + - 0.56548226 + - -0.2485694 + - 0.07929731 + - 0.18481643 + - 0.87893856 + - -0.3125372 + - 0.058244053 + - -0.29463747 + - -0.7056966 + - -0.37935346 + - -0.021258175 + - -0.44287688 + - -0.2136155 + - -0.08663365 + - -0.26825505 + - -0.16857736 + - -0.09559265 + - -0.39557096 + - -0.41658902 + - 0.3676616 + - -0.64672923 + - -0.12855792 + - 0.08793992 + - -0.25502628 + - -0.098317705 + - -0.15450214 + - 0.4101665 + - 0.6836288 + - 0.36123115 + - -0.53651035 + - -0.1711915 + - 0.084756374 + - 0.30335525 + - -0.085726015 + - -0.048317958 + - -0.24110518 + - 0.05647908 + - -0.019259367 + - 0.17239684 + - 0.39402467 + - -0.10658203 + - -0.1123959 + - -0.16120604 + - 0.20132384 + - 0.08628984 + - -0.4692899 + - -0.12350084 + - -0.000000000000000000000000000000013798718 + - -0.23723039 + - -0.028244663 + - 0.1483249 + - 0.13795137 + - 0.13788892 + - -0.41106498 + - -0.18105787 + - 0.01951392 + - -0.46009332 + - -0.020349242 + - -0.56887496 + - 0.43703493 + - -0.35521477 + - 0.4935305 + - 0.55461323 + - -0.05129622 + - -0.26033637 + - 0.7802168 + - 0.14747728 + - -0.6619525 + - -0.07878506 + - -0.2758619 + - -0.32319492 + - -0.11061615 + - -0.38868052 + - -0.3068285 + - 0.63651663 + - 0.098744065 + - 0.24740677 + - -0.043371066 + - -0.24581526 + - -0.060942523 + - -0.1463254 + - -0.03227488 + - 0.12888917 + - 0.14555049 + - 0.030782826 + - -0.34510422 + - 0.20764378 + - -0.2146532 + - -0.04867282 + - 0.29826757 + - -0.112927206 + - -0.422865 + - -0.10070534 + - 0.3019427 + - 0.4666604 + - -0.16918279 + - -0.53125834 + - -0.44115728 + - 0.31364805 + - -0.33498597 + - 0.044031385 + - -0.07271907 + - 0.4339783 + - 0.20187932 + - 0.32520926 + - 0.43127257 + - 0.60400313 + - 0.3258574 + - 0.2811687 + - 0.19900197 + - -0.24196737 + - 0.36056697 + - 0.091978036 + - 0.2377706 + - 0.072412215 + - 0.65998733 + - 0.44982168 + - 0.09455937 + - -0.1764154 + - 0.09192574 + - 0.09651725 + - -0.6582726 + - 0.19179328 + - 0.121408895 + - 0.20676559 + - -0.3042583 + - -0.3223266 + - 0.46790707 + - -0.2339656 + - 0.5563017 + - -0.13776281 + - 0.07729397 + - -0.020787623 + - 0.060829278 + - -0.0035964996 + - -0.782571 + - 0.16309232 + - 0.56995416 + - -0.073728755 + - -0.006863065 + - 0.09237085 + - 0.040949203 + - -0.15616469 + - 0.000000000000000000000000000000010410091 + - -0.6627872 + - 0.22273433 + - -0.3764339 + - 0.48016137 + - -0.02975322 + - -0.014806971 + - -0.14955428 + - -0.17068936 + - -0.5620402 + - 0.1926989 + - 0.05246362 + - -0.0149945915 + - 0.017368153 + - -0.014105275 + - 0.30300784 + - 0.0820339 + - 0.10900688 + - -0.1758156 + - -0.2796759 + - -0.15521398 + - 0.082758754 + - 0.63070357 + - -0.37660056 + - 0.20716804 + - -0.14281407 + - 0.0015612431 + - -0.2793287 + - 0.4062658 + - -0.22189105 + - 0.3566959 + - 0.14430332 + - -0.17170164 + - -0.87296 + - 0.015776588 + - -0.07914089 + - -0.15467063 + - 0.33060408 + - -0.15340795 + - -0.5013988 + - -0.35655665 + - 0.20658799 + - 0.058578372 + - -0.46810097 + - 0.031425357 + - -0.31070766 + - -0.38097033 + - -0.038344827 + - 0.14072128 + - 0.16105229 + - 0.23263907 + - 0.11929658 + - -0.19015168 + - 0.03561537 + - -0.3415985 + - -0.044782728 + - -0.13689396 + - -0.055164635 + - 0.021532819 + - 0.12246307 + - 0.29629645 + - -0.5619918 + - -0.44489312 + - -0.14099455 + - -0.095405675 + - 0.43164128 + - 0.20916349 + - -0.3363283 + - 0.59958637 + - -0.27061725 + - 0.18687364 + - 0.7507555 + - 0.014387103 + - -0.19831714 + - 0.21728376 + - -0.70773613 + - -0.45452872 + - 0.034639467 + - 0.10750381 + - 0.024194703 + - -0.6197492 + - 0.49192038 + - -0.40235862 + - 0.22315523 + - 0.87580097 + - 0.35277522 + - 0.71300834 + - 0.09896057 + - -0.16204226 + - -0.2301983 + - -0.15976599 + - -0.44753385 + - 0.26244015 + - -0.25863582 + - 0.6495952 + - -0.46806747 + - -0.00000009278316 + - -0.05864382 + - -0.1562716 + - 0.5997297 + - -0.55825746 + - -0.0038222922 + - -0.16698924 + - 0.24631433 + - 0.5396025 + - -0.15040281 + - 0.3144548 + - 0.099244624 + - -0.24408191 + - -0.33079758 + - 0.44268113 + - -0.110842876 + - 0.35829172 + - 0.24510199 + - -0.02560123 + - -0.07240376 + - 0.24795437 + - 0.4385171 + - 0.44353202 + - -0.36453864 + - -0.11752972 + - 0.06036435 + - -0.29738513 + - -0.16857165 + - 0.31146476 + - 0.02234754 + - 0.46079713 + - -0.17784917 + - 0.4304716 + - 0.1359111 + - -0.050859615 + - 0.66032004 + - 0.43336856 + - 0.3441714 + - -0.09320268 + - -0.14670397 + - -0.2464672 + - -0.23819153 + - 0.78434765 + - -0.41557816 + - -0.101150885 + - -0.30333805 + - -0.088198826 + - 0.25737122 + - -0.1915979 + - 0.29874736 + - -0.18905388 + - -0.044644568 + - 0.45899832 + - 0.3363795 + - 0.10431914 + - 0.3146274 + - 0.0075275917 + - -0.1305914 + - -0.4539809 + - -0.4078899 + - 0.60822535 + - -0.26053923 + - 0.31067634 + - -0.06968038 + - -0.14750677 +- - -0.5904484 + - -0.25218195 + - 0.14959621 + - 0.05505526 + - -0.11826475 + - -0.16704129 + - -0.23318417 + - -0.0704903 + - -0.19961402 + - -0.2959458 + - -0.7729306 + - 0.14422165 + - -0.29086623 + - -0.18537273 + - -0.9093197 + - -0.019904744 + - 0.02856647 + - 0.34820673 + - -1.0033797 + - -0.41697913 + - 0.19350834 + - 0.40060228 + - -0.39103433 + - -0.18893544 + - 0.45299938 + - 0.25677818 + - 0.17712198 + - -0.3142255 + - 0.12515345 + - -0.08413245 + - 0.65004975 + - -0.23057863 + - 0.18415941 + - 0.4963523 + - -0.36540174 + - 0.83785826 + - -0.5513201 + - 0.103302784 + - 0.0940645 + - -0.21010701 + - -0.29487512 + - -0.18855686 + - -0.009985788 + - 0.18686627 + - 0.9258731 + - 0.33936834 + - -0.08204167 + - -0.60297924 + - 0.1529275 + - 0.2101926 + - -0.20914623 + - -0.13881606 + - -0.38299662 + - 0.63747364 + - 0.003919348 + - 0.098739825 + - 0.27915862 + - -0.021564007 + - -0.5371316 + - 0.32010522 + - 0.4855331 + - -0.69396293 + - 0.024767732 + - 0.23580678 + - 0.29508504 + - -0.27021644 + - 0.048729945 + - 0.110520415 + - 0.5391848 + - -0.771077 + - 0.42122838 + - 0.47483942 + - -0.052862015 + - 0.07978022 + - 0.14361086 + - -0.43833405 + - 0.21757023 + - 0.055354383 + - 0.74647045 + - -0.21855037 + - 0.7267293 + - 0.54959774 + - -0.11634091 + - 0.36827073 + - 1.0121477 + - -0.5172255 + - 0.117366515 + - 0.0698515 + - -0.67828286 + - -0.5236744 + - -0.27104068 + - -0.5252708 + - -0.59517235 + - -0.0011284138 + - -0.29228482 + - -0.09563696 + - -0.05197223 + - -0.9138813 + - -0.4042197 + - 0.38407257 + - -0.022372637 + - 0.06314764 + - 0.41756183 + - -0.04670214 + - -0.069010004 + - 0.090586014 + - 0.42655104 + - 0.10947996 + - 0.31035274 + - -0.62466854 + - -0.19219613 + - 0.29427573 + - 0.18328457 + - -0.18981433 + - 0.16418755 + - -0.19344743 + - -0.10209768 + - -0.04093083 + - -0.14816917 + - 0.69175416 + - -0.8734301 + - -0.05981831 + - -0.24518713 + - -0.12859611 + - -0.2916654 + - -0.18874489 + - -0.3669198 + - -0.000000000000000000000000000000029240375 + - -0.003938394 + - -0.36723635 + - -0.4536573 + - 0.049564976 + - 0.3200263 + - -0.4268046 + - -0.20275964 + - 0.093998395 + - -0.42062044 + - 0.44631353 + - -0.14178619 + - 0.0815359 + - -0.13849926 + - 1.1363466 + - 0.3480273 + - 0.09552129 + - -0.64421636 + - 0.5022697 + - 0.83456355 + - -0.5952269 + - -0.06076554 + - 0.31910786 + - -0.17580135 + - 0.022548726 + - -0.054858353 + - -0.10727251 + - 0.1562703 + - 0.16639505 + - 0.21630454 + - -0.028444165 + - -0.5627595 + - 0.07629048 + - -0.16489701 + - -0.069293454 + - 0.3651604 + - 0.042008024 + - 0.018344846 + - -0.0626462 + - 0.3005414 + - -0.2635724 + - 0.34574392 + - 0.062163908 + - -0.061937597 + - -0.43312663 + - -0.2066199 + - -0.1683005 + - 0.20767058 + - -0.71167225 + - -0.6564812 + - -0.38578632 + - -0.052831333 + - -0.3264928 + - -0.18351099 + - -0.4606121 + - 0.2933788 + - 0.2779899 + - 0.21287741 + - 0.206459 + - 0.12392608 + - 0.18230744 + - 0.53405255 + - 0.39990216 + - -0.46907288 + - 0.66455895 + - -0.09669055 + - 0.16171496 + - 0.6648889 + - 0.68079907 + - 1.0480419 + - -0.094723225 + - 0.02977555 + - 0.09691883 + - 0.20310333 + - -0.57307446 + - -0.07695443 + - 0.08091426 + - 0.15485184 + - -0.26107487 + - -0.25665718 + - 0.78575295 + - -0.47057983 + - 0.3093873 + - -0.25569737 + - 0.0062563634 + - -0.5304047 + - 0.60793275 + - -0.24611141 + - -0.3020915 + - 0.26118788 + - 0.01478196 + - -0.70216304 + - -0.049419917 + - -0.14812419 + - -0.07927934 + - -0.16526112 + - 0.000000000000000000000000000000021845427 + - -0.25161034 + - 0.4341398 + - -0.87574375 + - 0.71753454 + - 0.2905553 + - 0.007794529 + - -0.6243053 + - 0.35534343 + - -0.5528342 + - 0.18967059 + - 0.20288529 + - -0.035637587 + - 0.45268387 + - 0.109326735 + - 0.1914785 + - -0.16920124 + - -0.3563299 + - -0.34934732 + - -0.47009516 + - -0.26941064 + - 0.2352554 + - 0.5886043 + - -0.3380012 + - -0.24570635 + - -0.15959099 + - -0.22487848 + - -0.25580263 + - 0.12511626 + - -0.10618205 + - 0.2850696 + - 0.17198822 + - -0.32835695 + - -0.67826843 + - 0.44815382 + - 0.2926607 + - 0.50614864 + - 0.59291375 + - -0.28529674 + - -0.2626424 + - -0.42411417 + - 0.21714236 + - -0.009686342 + - 0.18806958 + - 0.2543909 + - -0.31932625 + - -0.058950584 + - 0.0066510993 + - -0.043817382 + - 0.32712084 + - -0.19686058 + - 0.31065568 + - 0.027757645 + - 0.034487057 + - -0.35713312 + - -0.25857708 + - -0.07837208 + - -0.10901906 + - 0.18711178 + - 0.14383718 + - 0.43526158 + - -0.18126877 + - -0.4950599 + - -0.116322875 + - -0.18074515 + - -0.16935274 + - 0.805221 + - -0.59641445 + - 0.62855345 + - 0.032347683 + - 0.14520326 + - 0.809142 + - 0.108584665 + - 0.34700632 + - 0.580749 + - -0.53957874 + - -0.5434822 + - 0.05687144 + - -0.09498585 + - -0.33424398 + - -0.18342555 + - 0.39793792 + - -0.8092169 + - -0.10401372 + - 0.65306324 + - -0.09104957 + - 0.10358218 + - -0.24646153 + - 0.5131639 + - -0.35961035 + - -0.026082557 + - -0.24173585 + - 0.10468064 + - -0.63350576 + - 0.0022953025 + - -0.52731735 + - -0.000000091965795 + - -0.19923782 + - 0.08858619 + - 0.5564927 + - -0.80046386 + - 0.30863148 + - -0.18751955 + - 0.10183903 + - 0.82070285 + - 0.024335938 + - 0.07749426 + - 0.60264343 + - 0.014298631 + - -0.5296712 + - -0.15352051 + - 0.026842814 + - 0.61098564 + - 0.459986 + - -0.00048416216 + - 0.4151469 + - 0.024833662 + - 0.8714803 + - 0.3206982 + - 0.12014544 + - -0.055944566 + - 0.22077613 + - -0.6302576 + - 0.36912346 + - 0.680718 + - 0.106342845 + - -0.4253362 + - -0.20274441 + - 0.5338388 + - -0.1241984 + - -0.1653124 + - 0.019658208 + - 0.88533986 + - 0.3077552 + - -0.28941244 + - -0.058864314 + - -0.40539756 + - -0.4040249 + - 1.13756 + - 0.2780994 + - -0.06830321 + - -0.46814448 + - 0.089857385 + - 0.31686476 + - -0.054425247 + - 0.14721735 + - 0.11701803 + - -0.16131987 + - 0.5147092 + - 0.6204316 + - 0.64212894 + - -0.036693208 + - 0.1190122 + - 0.017730946 + - -0.5259076 + - -0.16212647 + - 0.7366949 + - -0.07890918 + - 0.54164237 + - 0.28229737 + - 0.27705735 + diff --git a/backends/candle/tests/snapshots/test_bert__mini_single.snap b/backends/candle/tests/snapshots/test_bert__mini_single.snap new file mode 100644 index 00000000..cc2437fd --- /dev/null +++ b/backends/candle/tests/snapshots/test_bert__mini_single.snap @@ -0,0 +1,389 @@ +--- +source: backends/candle/tests/test_bert.rs +expression: embeddings_single +--- +- - -0.5905822 + - -0.2525167 + - 0.14959584 + - 0.054548685 + - -0.11789139 + - -0.16702051 + - -0.2331886 + - -0.07111306 + - -0.19944426 + - -0.29595104 + - -0.7731143 + - 0.14407897 + - -0.29080537 + - -0.18528816 + - -0.90925914 + - -0.020204756 + - 0.028502962 + - 0.34817806 + - -1.0032953 + - -0.41697448 + - 0.1934247 + - 0.40048108 + - -0.3906826 + - -0.18889977 + - 0.4526494 + - 0.25689012 + - 0.17726377 + - -0.31426263 + - 0.1250183 + - -0.08393615 + - 0.6500511 + - -0.2302905 + - 0.18419918 + - 0.49622348 + - -0.36559528 + - 0.83769214 + - -0.551407 + - 0.103570856 + - 0.09400676 + - -0.21002762 + - -0.29478115 + - -0.18860105 + - -0.009910335 + - 0.18697806 + - 0.92610204 + - 0.33941612 + - -0.082030125 + - -0.6034355 + - 0.15340401 + - 0.21050398 + - -0.20921578 + - -0.13902524 + - -0.3827591 + - 0.63712025 + - 0.0040485496 + - 0.09897916 + - 0.2789379 + - -0.021972388 + - -0.53686434 + - 0.3202223 + - 0.48562184 + - -0.69386727 + - 0.024683978 + - 0.23594321 + - 0.29521427 + - -0.27045968 + - 0.048694637 + - 0.110244274 + - 0.53926224 + - -0.77139634 + - 0.4209654 + - 0.4750518 + - -0.052538875 + - 0.07966832 + - 0.14345014 + - -0.43797436 + - 0.21794893 + - 0.055310573 + - 0.7462658 + - -0.21850769 + - 0.7266181 + - 0.549555 + - -0.11633466 + - 0.368249 + - 1.0122206 + - -0.5168573 + - 0.11747182 + - 0.06994404 + - -0.6778618 + - -0.52381235 + - -0.2710965 + - -0.5250548 + - -0.5948887 + - -0.0012941743 + - -0.2924962 + - -0.095263354 + - -0.051451437 + - -0.9137996 + - -0.40414217 + - 0.3842635 + - -0.0225345 + - 0.062937684 + - 0.41753647 + - -0.047207624 + - -0.068914846 + - 0.090506375 + - 0.42675138 + - 0.109467186 + - 0.31070897 + - -0.62438476 + - -0.19247803 + - 0.2941215 + - 0.18347324 + - -0.18997712 + - 0.1642261 + - -0.19337931 + - -0.102235556 + - -0.041086674 + - -0.14846203 + - 0.6917226 + - -0.8732191 + - -0.060058128 + - -0.24552263 + - -0.12856898 + - -0.2914501 + - -0.1887479 + - -0.3668236 + - -0.000000000000000000000000000000029240537 + - -0.0039896094 + - -0.36697504 + - -0.45366925 + - 0.04975644 + - 0.31989673 + - -0.42694908 + - -0.20267479 + - 0.09384082 + - -0.42025587 + - 0.44595543 + - -0.14169464 + - 0.0813049 + - -0.138726 + - 1.1364049 + - 0.3478461 + - 0.09553712 + - -0.64425725 + - 0.5023483 + - 0.83466536 + - -0.5949079 + - -0.060975213 + - 0.31882063 + - -0.17581889 + - 0.022465851 + - -0.05469214 + - -0.10731493 + - 0.15609773 + - 0.16630125 + - 0.21610661 + - -0.028483214 + - -0.5626438 + - 0.07620309 + - -0.16477105 + - -0.069259115 + - 0.3652963 + - 0.042083118 + - 0.018148389 + - -0.062457535 + - 0.30002403 + - -0.26324186 + - 0.3456572 + - 0.062287714 + - -0.06215567 + - -0.43306065 + - -0.20688285 + - -0.1685666 + - 0.2078495 + - -0.71178585 + - -0.6564981 + - -0.38591677 + - -0.052962523 + - -0.3261991 + - -0.18351372 + - -0.4604339 + - 0.2930664 + - 0.277802 + - 0.21312878 + - 0.20633987 + - 0.12381153 + - 0.18202356 + - 0.53390193 + - 0.39961782 + - -0.46880856 + - 0.6647521 + - -0.09672223 + - 0.16156423 + - 0.6650926 + - 0.6806067 + - 1.0482094 + - -0.09435779 + - 0.029582651 + - 0.09683895 + - 0.203073 + - -0.5732891 + - -0.07686127 + - 0.08099214 + - 0.15478149 + - -0.26159474 + - -0.2566363 + - 0.78577054 + - -0.47089925 + - 0.30936465 + - -0.25532243 + - 0.0064516324 + - -0.53040725 + - 0.6080942 + - -0.24619754 + - -0.30199987 + - 0.26108736 + - 0.014690906 + - -0.70210713 + - -0.049562275 + - -0.14834751 + - -0.07956274 + - -0.16515985 + - 0.000000000000000000000000000000021861 + - -0.25157446 + - 0.43375129 + - -0.8755244 + - 0.7175831 + - 0.2902798 + - 0.0074201175 + - -0.62435234 + - 0.35527846 + - -0.552679 + - 0.18954086 + - 0.20335351 + - -0.035552207 + - 0.4529572 + - 0.10931892 + - 0.19148143 + - -0.16911459 + - -0.35587427 + - -0.3493578 + - -0.47012717 + - -0.26946193 + - 0.23536286 + - 0.58844364 + - -0.3380215 + - -0.24592063 + - -0.15954623 + - -0.22448906 + - -0.25596026 + - 0.12521902 + - -0.106268756 + - 0.28519255 + - 0.17178701 + - -0.3287186 + - -0.6783996 + - 0.4482324 + - 0.29271346 + - 0.50640035 + - 0.5931688 + - -0.28495908 + - -0.26254502 + - -0.4237756 + - 0.21733634 + - -0.009652589 + - 0.18802977 + - 0.254598 + - -0.31955296 + - -0.05901348 + - 0.0068638837 + - -0.044022094 + - 0.32722753 + - -0.19676164 + - 0.31067184 + - 0.027737487 + - 0.034569282 + - -0.3569302 + - -0.25845194 + - -0.07841165 + - -0.10891752 + - 0.18727538 + - 0.14427944 + - 0.43520635 + - -0.18098663 + - -0.4947866 + - -0.11612529 + - -0.18040998 + - -0.16923454 + - 0.8050995 + - -0.596841 + - 0.6288896 + - 0.03233897 + - 0.14519283 + - 0.80883557 + - 0.10845174 + - 0.34680286 + - 0.5812995 + - -0.5397416 + - -0.5434021 + - 0.057071857 + - -0.09486484 + - -0.33432207 + - -0.18326911 + - 0.39803055 + - -0.80921376 + - -0.10375113 + - 0.65296334 + - -0.09063423 + - 0.10393066 + - -0.24666296 + - 0.5133282 + - -0.35965517 + - -0.02616671 + - -0.24137843 + - 0.10464253 + - -0.6334521 + - 0.0021675143 + - -0.52741486 + - -0.00000009196107 + - -0.19908127 + - 0.088920064 + - 0.5567359 + - -0.8009427 + - 0.3083997 + - -0.18737307 + - 0.10197395 + - 0.82060134 + - 0.024411181 + - 0.07724148 + - 0.60279316 + - 0.01429141 + - -0.5294417 + - -0.15347035 + - 0.027200898 + - 0.6112227 + - 0.45954376 + - -0.0004197508 + - 0.41491097 + - 0.02481507 + - 0.8714842 + - 0.32075447 + - 0.120130405 + - -0.055921085 + - 0.22076562 + - -0.6300306 + - 0.36908314 + - 0.68083143 + - 0.10630507 + - -0.4252642 + - -0.20267203 + - 0.53380257 + - -0.12403769 + - -0.16542706 + - 0.019964132 + - 0.8855693 + - 0.30768922 + - -0.28937826 + - -0.05903546 + - -0.4054046 + - -0.4042628 + - 1.1373893 + - 0.27809668 + - -0.06820501 + - -0.4680714 + - 0.089684784 + - 0.31660247 + - -0.054611325 + - 0.14755571 + - 0.11694164 + - -0.16122253 + - 0.51518285 + - 0.6203127 + - 0.64230794 + - -0.036301676 + - 0.11894374 + - 0.017895553 + - -0.5259501 + - -0.16221896 + - 0.73642933 + - -0.079223275 + - 0.54166216 + - 0.28228587 + - 0.27675694 + diff --git a/backends/candle/tests/snapshots/test_flash_bert__emotions_batch.snap b/backends/candle/tests/snapshots/test_flash_bert__emotions_batch.snap new file mode 100644 index 00000000..486b9ca0 --- /dev/null +++ b/backends/candle/tests/snapshots/test_flash_bert__emotions_batch.snap @@ -0,0 +1,89 @@ +--- +source: backends/candle/tests/test_flash_bert.rs +expression: predictions_batch +--- +- - -6.5507813 + - -6.3007813 + - -4.8671875 + - -3.9589844 + - -4.6328125 + - -6.28125 + - -6.0703125 + - -5.7773438 + - -6.9257813 + - -6.1875 + - -5.671875 + - -6.1679688 + - -7.5117188 + - -6.8671875 + - -7.1835938 + - -7.1289063 + - -8.2109375 + - -7.015625 + - -7.1328125 + - -8.53125 + - -6.2304688 + - -8.7421875 + - -5.7773438 + - -8.65625 + - -8.2578125 + - -6.6484375 + - -7.40625 + - 3.046875 +- - -5.8164063 + - -6.6132813 + - -5.2773438 + - -2.6328125 + - -4.640625 + - -5.578125 + - -5.8007813 + - -6.03125 + - -5.8710938 + - 0.45166016 + - -3.0253906 + - -5.3945313 + - -5.1875 + - -6.265625 + - -6.1992188 + - -6.9804688 + - -5.5664063 + - -5.5195313 + - -5.890625 + - -4.8710938 + - -5.625 + - -7.609375 + - -4.3867188 + - -6.0390625 + - -4.9375 + - -0.6699219 + - -6.4023438 + - -2.4492188 +- - -6.5507813 + - -6.3007813 + - -4.8671875 + - -3.9589844 + - -4.6328125 + - -6.28125 + - -6.0703125 + - -5.7773438 + - -6.9257813 + - -6.1875 + - -5.671875 + - -6.1679688 + - -7.5117188 + - -6.8671875 + - -7.1835938 + - -7.1289063 + - -8.2109375 + - -7.015625 + - -7.1328125 + - -8.53125 + - -6.2304688 + - -8.7421875 + - -5.7773438 + - -8.65625 + - -8.2578125 + - -6.6484375 + - -7.40625 + - 3.046875 + diff --git a/backends/candle/tests/snapshots/test_flash_bert__emotions_single.snap b/backends/candle/tests/snapshots/test_flash_bert__emotions_single.snap new file mode 100644 index 00000000..d4503c42 --- /dev/null +++ b/backends/candle/tests/snapshots/test_flash_bert__emotions_single.snap @@ -0,0 +1,33 @@ +--- +source: backends/candle/tests/test_flash_bert.rs +expression: predictions_single +--- +- - -6.546875 + - -6.3007813 + - -4.8671875 + - -3.9589844 + - -4.6328125 + - -6.28125 + - -6.0703125 + - -5.7773438 + - -6.9257813 + - -6.1914063 + - -5.671875 + - -6.1679688 + - -7.5117188 + - -6.8671875 + - -7.1835938 + - -7.1289063 + - -8.2109375 + - -7.015625 + - -7.1328125 + - -8.53125 + - -6.2304688 + - -8.7421875 + - -5.7773438 + - -8.65625 + - -8.2578125 + - -6.6484375 + - -7.40625 + - 3.046875 + diff --git a/backends/candle/tests/snapshots/test_flash_bert__mini_batch.snap b/backends/candle/tests/snapshots/test_flash_bert__mini_batch.snap new file mode 100644 index 00000000..0cb43472 --- /dev/null +++ b/backends/candle/tests/snapshots/test_flash_bert__mini_batch.snap @@ -0,0 +1,1157 @@ +--- +source: backends/candle/tests/test_flash_bert.rs +expression: embeddings_batch +--- +- - -0.58984375 + - -0.25219727 + - 0.14880371 + - 0.05505371 + - -0.11785889 + - -0.16687012 + - -0.23266602 + - -0.0713501 + - -0.19946289 + - -0.29589844 + - -0.7734375 + - 0.14379883 + - -0.29101563 + - -0.18518066 + - -0.9091797 + - -0.020019531 + - 0.028320313 + - 0.34765625 + - -1.0029297 + - -0.41674805 + - 0.19372559 + - 0.4008789 + - -0.39111328 + - -0.18908691 + - 0.45239258 + - 0.2565918 + - 0.17712402 + - -0.31420898 + - 0.125 + - -0.083984375 + - 0.64990234 + - -0.23046875 + - 0.18432617 + - 0.49609375 + - -0.36523438 + - 0.83740234 + - -0.55078125 + - 0.103881836 + - 0.09411621 + - -0.21032715 + - -0.29467773 + - -0.18859863 + - -0.009727478 + - 0.18688965 + - 0.92626953 + - 0.33911133 + - -0.08227539 + - -0.60253906 + - 0.15332031 + - 0.21032715 + - -0.20922852 + - -0.13903809 + - -0.38330078 + - 0.63720703 + - 0.0040779114 + - 0.09838867 + - 0.27856445 + - -0.02142334 + - -0.5361328 + - 0.3203125 + - 0.48535156 + - -0.6933594 + - 0.024475098 + - 0.23571777 + - 0.29492188 + - -0.2705078 + - 0.048706055 + - 0.11047363 + - 0.5390625 + - -0.7709961 + - 0.4206543 + - 0.47485352 + - -0.052856445 + - 0.07922363 + - 0.14367676 + - -0.43798828 + - 0.21789551 + - 0.055267334 + - 0.74658203 + - -0.21789551 + - 0.7265625 + - 0.55029297 + - -0.115356445 + - 0.36865234 + - 1.0117188 + - -0.51660156 + - 0.1171875 + - 0.06982422 + - -0.6772461 + - -0.5234375 + - -0.27075195 + - -0.52490234 + - -0.59521484 + - -0.00090646744 + - -0.29223633 + - -0.095581055 + - -0.051849365 + - -0.9140625 + - -0.4033203 + - 0.38427734 + - -0.022644043 + - 0.06329346 + - 0.4169922 + - -0.046691895 + - -0.0690918 + - 0.090148926 + - 0.42626953 + - 0.10913086 + - 0.31054688 + - -0.6245117 + - -0.19177246 + - 0.2939453 + - 0.18359375 + - -0.18981934 + - 0.16455078 + - -0.19396973 + - -0.10266113 + - -0.041015625 + - -0.14880371 + - 0.69140625 + - -0.87353516 + - -0.059570313 + - -0.2454834 + - -0.12854004 + - -0.29174805 + - -0.18908691 + - -0.3671875 + - 0 + - -0.0034694672 + - -0.36645508 + - -0.453125 + - 0.049987793 + - 0.31860352 + - -0.4267578 + - -0.20263672 + - 0.09375 + - -0.4206543 + - 0.44604492 + - -0.14123535 + - 0.08166504 + - -0.13867188 + - 1.1357422 + - 0.3474121 + - 0.095825195 + - -0.64404297 + - 0.5019531 + - 0.83447266 + - -0.59472656 + - -0.061340332 + - 0.31933594 + - -0.17626953 + - 0.022628784 + - -0.05432129 + - -0.10723877 + - 0.15563965 + - 0.16589355 + - 0.21643066 + - -0.028274536 + - -0.5629883 + - 0.076660156 + - -0.16418457 + - -0.06933594 + - 0.3647461 + - 0.04208374 + - 0.017608643 + - -0.062194824 + - 0.30004883 + - -0.26367188 + - 0.34570313 + - 0.062072754 + - -0.06188965 + - -0.43286133 + - -0.20629883 + - -0.16833496 + - 0.20776367 + - -0.71191406 + - -0.65625 + - -0.38549805 + - -0.05316162 + - -0.32641602 + - -0.18273926 + - -0.46020508 + - 0.29223633 + - 0.2775879 + - 0.2133789 + - 0.20581055 + - 0.123535156 + - 0.1817627 + - 0.5332031 + - 0.3996582 + - -0.46826172 + - 0.6645508 + - -0.09686279 + - 0.16113281 + - 0.66503906 + - 0.6796875 + - 1.0478516 + - -0.09429932 + - 0.029663086 + - 0.09698486 + - 0.203125 + - -0.57373047 + - -0.07672119 + - 0.080566406 + - 0.15478516 + - -0.26049805 + - -0.25683594 + - 0.78564453 + - -0.4699707 + - 0.30908203 + - -0.25512695 + - 0.0063476563 + - -0.52978516 + - 0.6074219 + - -0.24633789 + - -0.30200195 + - 0.2614746 + - 0.014785767 + - -0.70166016 + - -0.049835205 + - -0.14868164 + - -0.07977295 + - -0.16442871 + - 0 + - -0.25097656 + - 0.43359375 + - -0.87597656 + - 0.71728516 + - 0.2902832 + - 0.0073280334 + - -0.625 + - 0.35424805 + - -0.5522461 + - 0.18920898 + - 0.20275879 + - -0.03604126 + - 0.45239258 + - 0.10925293 + - 0.19152832 + - -0.16906738 + - -0.35546875 + - -0.34936523 + - -0.47070313 + - -0.2692871 + - 0.23535156 + - 0.5883789 + - -0.33813477 + - -0.24609375 + - -0.15979004 + - -0.22485352 + - -0.25512695 + - 0.12512207 + - -0.10601807 + - 0.2854004 + - 0.17150879 + - -0.32885742 + - -0.67822266 + - 0.44750977 + - 0.29223633 + - 0.50683594 + - 0.59228516 + - -0.28442383 + - -0.26293945 + - -0.42382813 + - 0.21789551 + - -0.009536743 + - 0.18823242 + - 0.25463867 + - -0.3190918 + - -0.05886841 + - 0.007217407 + - -0.04397583 + - 0.32739258 + - -0.19689941 + - 0.31079102 + - 0.027145386 + - 0.034210205 + - -0.3564453 + - -0.25854492 + - -0.07788086 + - -0.108947754 + - 0.18701172 + - 0.14440918 + - 0.43554688 + - -0.1809082 + - -0.49536133 + - -0.1159668 + - -0.18029785 + - -0.16955566 + - 0.8046875 + - -0.59716797 + - 0.62841797 + - 0.032073975 + - 0.14501953 + - 0.80859375 + - 0.10845947 + - 0.34765625 + - 0.58154297 + - -0.54003906 + - -0.5439453 + - 0.056793213 + - -0.0949707 + - -0.33447266 + - -0.18261719 + - 0.39770508 + - -0.80908203 + - -0.10369873 + - 0.65283203 + - -0.09039307 + - 0.10418701 + - -0.24621582 + - 0.5131836 + - -0.35986328 + - -0.026062012 + - -0.24121094 + - 0.10449219 + - -0.6328125 + - 0.0012207031 + - -0.52734375 + - -0.000000059604645 + - -0.19897461 + - 0.08880615 + - 0.55615234 + - -0.7998047 + - 0.3083496 + - -0.18652344 + - 0.10241699 + - 0.8203125 + - 0.024841309 + - 0.07824707 + - 0.6015625 + - 0.013633728 + - -0.5292969 + - -0.15368652 + - 0.027313232 + - 0.61083984 + - 0.45922852 + - -0.0005578995 + - 0.41455078 + - 0.024658203 + - 0.87109375 + - 0.32080078 + - 0.119506836 + - -0.055786133 + - 0.22033691 + - -0.62939453 + - 0.36938477 + - 0.68066406 + - 0.10620117 + - -0.42504883 + - -0.20227051 + - 0.5336914 + - -0.12432861 + - -0.16491699 + - 0.019805908 + - 0.8852539 + - 0.30737305 + - -0.28930664 + - -0.058654785 + - -0.40551758 + - -0.40454102 + - 1.1376953 + - 0.2783203 + - -0.068115234 + - -0.46801758 + - 0.09051514 + - 0.3166504 + - -0.053985596 + - 0.14758301 + - 0.11694336 + - -0.16125488 + - 0.51464844 + - 0.62060547 + - 0.64208984 + - -0.035888672 + - 0.118896484 + - 0.017715454 + - -0.52490234 + - -0.16210938 + - 0.7363281 + - -0.078552246 + - 0.5415039 + - 0.28173828 + - 0.2770996 +- - -0.118896484 + - -0.58447266 + - 0.3466797 + - -0.24951172 + - -0.3317871 + - -0.20483398 + - -0.14587402 + - -0.21459961 + - 0.091918945 + - -0.24926758 + - -0.27905273 + - 0.24719238 + - -0.29785156 + - 0.31469727 + - -0.5292969 + - 0.006164551 + - 0.19006348 + - 0.16772461 + - -0.8251953 + - -0.26513672 + - -0.17529297 + - 0.22070313 + - -0.16442871 + - -0.3701172 + - 0.5703125 + - 0.32006836 + - 0.29785156 + - -0.27661133 + - 0.15454102 + - -0.4091797 + - 0.29589844 + - 0.15844727 + - -0.18554688 + - 0.1796875 + - -0.14355469 + - 0.40283203 + - -0.20532227 + - 0.13635254 + - -0.020446777 + - -0.00010681152 + - -0.16699219 + - 0.07495117 + - -0.066223145 + - 0.0063323975 + - 0.45361328 + - 0.1998291 + - -0.09301758 + - -0.3413086 + - -0.02935791 + - 0.039398193 + - -0.41601563 + - -0.08917236 + - -0.13879395 + - 0.09564209 + - -0.031982422 + - -0.021026611 + - 0.53759766 + - -0.27539063 + - -0.24938965 + - 0.38964844 + - -0.034301758 + - -0.20458984 + - 0.00415802 + - 0.59521484 + - 0.20153809 + - 0.34179688 + - 0.1809082 + - 0.14465332 + - 0.12097168 + - 0.039367676 + - 0.43701172 + - 0.4404297 + - -0.0004272461 + - 0.6328125 + - 0.13500977 + - 0.3544922 + - 0.48779297 + - 0.25610352 + - 0.5029297 + - 0.032806396 + - 0.5654297 + - -0.2487793 + - 0.07946777 + - 0.18481445 + - 0.87890625 + - -0.31225586 + - 0.058380127 + - -0.2944336 + - -0.70458984 + - -0.37939453 + - -0.021911621 + - -0.44311523 + - -0.21386719 + - -0.08691406 + - -0.26831055 + - -0.16894531 + - -0.095581055 + - -0.3955078 + - -0.4165039 + - 0.3671875 + - -0.6464844 + - -0.12854004 + - 0.087402344 + - -0.25439453 + - -0.09838867 + - -0.15478516 + - 0.4099121 + - 0.68359375 + - 0.36083984 + - -0.5361328 + - -0.1706543 + - 0.0847168 + - 0.30395508 + - -0.08605957 + - -0.04815674 + - -0.24145508 + - 0.05630493 + - -0.019165039 + - 0.171875 + - 0.39453125 + - -0.10644531 + - -0.11279297 + - -0.16113281 + - 0.20031738 + - 0.086242676 + - -0.47021484 + - -0.123168945 + - 0 + - -0.23730469 + - -0.028137207 + - 0.14819336 + - 0.1381836 + - 0.13745117 + - -0.4111328 + - -0.18139648 + - 0.019515991 + - -0.46020508 + - -0.019958496 + - -0.56933594 + - 0.43701172 + - -0.35595703 + - 0.49316406 + - 0.55371094 + - -0.05114746 + - -0.25976563 + - 0.78125 + - 0.14746094 + - -0.6621094 + - -0.0793457 + - -0.27612305 + - -0.32373047 + - -0.11004639 + - -0.38842773 + - -0.3071289 + - 0.63623047 + - 0.09869385 + - 0.24731445 + - -0.043640137 + - -0.24658203 + - -0.06085205 + - -0.14587402 + - -0.032196045 + - 0.12866211 + - 0.1459961 + - 0.030761719 + - -0.34472656 + - 0.20776367 + - -0.21508789 + - -0.048858643 + - 0.29833984 + - -0.11279297 + - -0.42285156 + - -0.10070801 + - 0.30126953 + - 0.46728516 + - -0.16931152 + - -0.53027344 + - -0.44189453 + - 0.31347656 + - -0.33447266 + - 0.04458618 + - -0.072509766 + - 0.43359375 + - 0.20166016 + - 0.3251953 + - 0.43139648 + - 0.6044922 + - 0.32617188 + - 0.28149414 + - 0.19836426 + - -0.24182129 + - 0.36010742 + - 0.091674805 + - 0.23718262 + - 0.072509766 + - 0.66015625 + - 0.4501953 + - 0.09399414 + - -0.17626953 + - 0.092163086 + - 0.09637451 + - -0.6586914 + - 0.19213867 + - 0.12145996 + - 0.20690918 + - -0.30444336 + - -0.3227539 + - 0.46826172 + - -0.23376465 + - 0.5566406 + - -0.1381836 + - 0.07788086 + - -0.02053833 + - 0.060546875 + - -0.0037231445 + - -0.78271484 + - 0.16271973 + - 0.5708008 + - -0.07342529 + - -0.006652832 + - 0.09222412 + - 0.040893555 + - -0.15576172 + - 0 + - -0.6621094 + - 0.22229004 + - -0.37695313 + - 0.4794922 + - -0.02935791 + - -0.014831543 + - -0.15063477 + - -0.17089844 + - -0.5620117 + - 0.1928711 + - 0.052124023 + - -0.015045166 + - 0.01751709 + - -0.014099121 + - 0.30322266 + - 0.082214355 + - 0.10925293 + - -0.17565918 + - -0.27978516 + - -0.1550293 + - 0.08282471 + - 0.6308594 + - -0.3774414 + - 0.20654297 + - -0.14282227 + - 0.0016174316 + - -0.2788086 + - 0.40649414 + - -0.22167969 + - 0.35668945 + - 0.14453125 + - -0.17211914 + - -0.87402344 + - 0.0154418945 + - -0.07891846 + - -0.1550293 + - 0.3305664 + - -0.15368652 + - -0.50097656 + - -0.35595703 + - 0.20654297 + - 0.05847168 + - -0.46826172 + - 0.031433105 + - -0.31103516 + - -0.38085938 + - -0.03805542 + - 0.13977051 + - 0.1616211 + - 0.23266602 + - 0.12005615 + - -0.19018555 + - 0.035339355 + - -0.3408203 + - -0.04522705 + - -0.13623047 + - -0.05517578 + - 0.021362305 + - 0.122924805 + - 0.29663086 + - -0.5620117 + - -0.44458008 + - -0.14135742 + - -0.095703125 + - 0.43164063 + - 0.20922852 + - -0.33642578 + - 0.5996094 + - -0.2709961 + - 0.18713379 + - 0.7504883 + - 0.014709473 + - -0.19873047 + - 0.2175293 + - -0.7080078 + - -0.45483398 + - 0.03451538 + - 0.10797119 + - 0.024139404 + - -0.6191406 + - 0.49316406 + - -0.4020996 + - 0.22387695 + - 0.87646484 + - 0.35351563 + - 0.7138672 + - 0.099487305 + - -0.16210938 + - -0.23095703 + - -0.16003418 + - -0.44750977 + - 0.26220703 + - -0.2590332 + - 0.6503906 + - -0.46875 + - -0.000000059604645 + - -0.05886841 + - -0.15698242 + - 0.5991211 + - -0.55810547 + - -0.00415802 + - -0.16674805 + - 0.24658203 + - 0.5390625 + - -0.15014648 + - 0.31445313 + - 0.099243164 + - -0.2446289 + - -0.3305664 + - 0.44360352 + - -0.11071777 + - 0.35888672 + - 0.24389648 + - -0.025939941 + - -0.072631836 + - 0.2475586 + - 0.43896484 + - 0.44384766 + - -0.36547852 + - -0.11798096 + - 0.060333252 + - -0.29736328 + - -0.16870117 + - 0.31201172 + - 0.021728516 + - 0.4609375 + - -0.17700195 + - 0.43017578 + - 0.1361084 + - -0.050842285 + - 0.66015625 + - 0.43408203 + - 0.34350586 + - -0.0927124 + - -0.14648438 + - -0.24645996 + - -0.23815918 + - 0.78515625 + - -0.41503906 + - -0.10083008 + - -0.30371094 + - -0.088012695 + - 0.25732422 + - -0.19091797 + - 0.2993164 + - -0.18933105 + - -0.04486084 + - 0.45922852 + - 0.33642578 + - 0.10498047 + - 0.3149414 + - 0.0077667236 + - -0.13061523 + - -0.45410156 + - -0.40771484 + - 0.60839844 + - -0.26049805 + - 0.31103516 + - -0.07019043 + - -0.14758301 +- - -0.58984375 + - -0.25219727 + - 0.14880371 + - 0.05505371 + - -0.11785889 + - -0.16687012 + - -0.23266602 + - -0.0713501 + - -0.19946289 + - -0.29589844 + - -0.7734375 + - 0.14379883 + - -0.29101563 + - -0.18518066 + - -0.9091797 + - -0.020019531 + - 0.028320313 + - 0.34765625 + - -1.0029297 + - -0.41674805 + - 0.19372559 + - 0.4008789 + - -0.39111328 + - -0.18908691 + - 0.45239258 + - 0.2565918 + - 0.17712402 + - -0.31420898 + - 0.125 + - -0.083984375 + - 0.64990234 + - -0.23046875 + - 0.18432617 + - 0.49609375 + - -0.36523438 + - 0.83740234 + - -0.55078125 + - 0.103881836 + - 0.09411621 + - -0.21032715 + - -0.29467773 + - -0.18859863 + - -0.009727478 + - 0.18688965 + - 0.92626953 + - 0.33911133 + - -0.08227539 + - -0.60253906 + - 0.15332031 + - 0.21032715 + - -0.20922852 + - -0.13903809 + - -0.38330078 + - 0.63720703 + - 0.0040779114 + - 0.09838867 + - 0.27856445 + - -0.02142334 + - -0.5361328 + - 0.3203125 + - 0.48535156 + - -0.6933594 + - 0.024475098 + - 0.23571777 + - 0.29492188 + - -0.2705078 + - 0.048706055 + - 0.11047363 + - 0.5390625 + - -0.7709961 + - 0.4206543 + - 0.47485352 + - -0.052856445 + - 0.07922363 + - 0.14367676 + - -0.43798828 + - 0.21789551 + - 0.055267334 + - 0.74658203 + - -0.21789551 + - 0.7265625 + - 0.55029297 + - -0.115356445 + - 0.36865234 + - 1.0117188 + - -0.51660156 + - 0.1171875 + - 0.06982422 + - -0.6772461 + - -0.5234375 + - -0.27075195 + - -0.52490234 + - -0.59521484 + - -0.00090646744 + - -0.29223633 + - -0.095581055 + - -0.051849365 + - -0.9140625 + - -0.4033203 + - 0.38427734 + - -0.022644043 + - 0.06329346 + - 0.4169922 + - -0.046691895 + - -0.0690918 + - 0.090148926 + - 0.42626953 + - 0.10913086 + - 0.31054688 + - -0.6245117 + - -0.19177246 + - 0.2939453 + - 0.18359375 + - -0.18981934 + - 0.16455078 + - -0.19396973 + - -0.10266113 + - -0.041015625 + - -0.14880371 + - 0.69140625 + - -0.87353516 + - -0.059570313 + - -0.2454834 + - -0.12854004 + - -0.29174805 + - -0.18908691 + - -0.3671875 + - 0 + - -0.0034694672 + - -0.36645508 + - -0.453125 + - 0.049987793 + - 0.31860352 + - -0.4267578 + - -0.20263672 + - 0.09375 + - -0.4206543 + - 0.44604492 + - -0.14123535 + - 0.08166504 + - -0.13867188 + - 1.1357422 + - 0.3474121 + - 0.095825195 + - -0.64404297 + - 0.5019531 + - 0.83447266 + - -0.59472656 + - -0.061340332 + - 0.31933594 + - -0.17626953 + - 0.022628784 + - -0.05432129 + - -0.10723877 + - 0.15563965 + - 0.16589355 + - 0.21643066 + - -0.028274536 + - -0.5629883 + - 0.076660156 + - -0.16418457 + - -0.06933594 + - 0.3647461 + - 0.04208374 + - 0.017608643 + - -0.062194824 + - 0.30004883 + - -0.26367188 + - 0.34570313 + - 0.062072754 + - -0.06188965 + - -0.43286133 + - -0.20629883 + - -0.16833496 + - 0.20776367 + - -0.71191406 + - -0.65625 + - -0.38549805 + - -0.05316162 + - -0.32641602 + - -0.18273926 + - -0.46020508 + - 0.29223633 + - 0.2775879 + - 0.2133789 + - 0.20581055 + - 0.123535156 + - 0.1817627 + - 0.5332031 + - 0.3996582 + - -0.46826172 + - 0.6645508 + - -0.09686279 + - 0.16113281 + - 0.66503906 + - 0.6796875 + - 1.0478516 + - -0.09429932 + - 0.029663086 + - 0.09698486 + - 0.203125 + - -0.57373047 + - -0.07672119 + - 0.080566406 + - 0.15478516 + - -0.26049805 + - -0.25683594 + - 0.78564453 + - -0.4699707 + - 0.30908203 + - -0.25512695 + - 0.0063476563 + - -0.52978516 + - 0.6074219 + - -0.24633789 + - -0.30200195 + - 0.2614746 + - 0.014785767 + - -0.70166016 + - -0.049835205 + - -0.14868164 + - -0.07977295 + - -0.16442871 + - 0 + - -0.25097656 + - 0.43359375 + - -0.87597656 + - 0.71728516 + - 0.2902832 + - 0.0073280334 + - -0.625 + - 0.35424805 + - -0.5522461 + - 0.18920898 + - 0.20275879 + - -0.03604126 + - 0.45239258 + - 0.10925293 + - 0.19152832 + - -0.16906738 + - -0.35546875 + - -0.34936523 + - -0.47070313 + - -0.2692871 + - 0.23535156 + - 0.5883789 + - -0.33813477 + - -0.24609375 + - -0.15979004 + - -0.22485352 + - -0.25512695 + - 0.12512207 + - -0.10601807 + - 0.2854004 + - 0.17150879 + - -0.32885742 + - -0.67822266 + - 0.44750977 + - 0.29223633 + - 0.50683594 + - 0.59228516 + - -0.28442383 + - -0.26293945 + - -0.42382813 + - 0.21789551 + - -0.009536743 + - 0.18823242 + - 0.25463867 + - -0.3190918 + - -0.05886841 + - 0.007217407 + - -0.04397583 + - 0.32739258 + - -0.19689941 + - 0.31079102 + - 0.027145386 + - 0.034210205 + - -0.3564453 + - -0.25854492 + - -0.07788086 + - -0.108947754 + - 0.18701172 + - 0.14440918 + - 0.43554688 + - -0.1809082 + - -0.49536133 + - -0.1159668 + - -0.18029785 + - -0.16955566 + - 0.8046875 + - -0.59716797 + - 0.62841797 + - 0.032073975 + - 0.14501953 + - 0.80859375 + - 0.10845947 + - 0.34765625 + - 0.58154297 + - -0.54003906 + - -0.5439453 + - 0.056793213 + - -0.0949707 + - -0.33447266 + - -0.18261719 + - 0.39770508 + - -0.80908203 + - -0.10369873 + - 0.65283203 + - -0.09039307 + - 0.10418701 + - -0.24621582 + - 0.5131836 + - -0.35986328 + - -0.026062012 + - -0.24121094 + - 0.10449219 + - -0.6328125 + - 0.0012207031 + - -0.52734375 + - -0.000000059604645 + - -0.19897461 + - 0.08880615 + - 0.55615234 + - -0.7998047 + - 0.3083496 + - -0.18652344 + - 0.10241699 + - 0.8203125 + - 0.024841309 + - 0.07824707 + - 0.6015625 + - 0.013633728 + - -0.5292969 + - -0.15368652 + - 0.027313232 + - 0.61083984 + - 0.45922852 + - -0.0005578995 + - 0.41455078 + - 0.024658203 + - 0.87109375 + - 0.32080078 + - 0.119506836 + - -0.055786133 + - 0.22033691 + - -0.62939453 + - 0.36938477 + - 0.68066406 + - 0.10620117 + - -0.42504883 + - -0.20227051 + - 0.5336914 + - -0.12432861 + - -0.16491699 + - 0.019805908 + - 0.8852539 + - 0.30737305 + - -0.28930664 + - -0.058654785 + - -0.40551758 + - -0.40454102 + - 1.1376953 + - 0.2783203 + - -0.068115234 + - -0.46801758 + - 0.09051514 + - 0.3166504 + - -0.053985596 + - 0.14758301 + - 0.11694336 + - -0.16125488 + - 0.51464844 + - 0.62060547 + - 0.64208984 + - -0.035888672 + - 0.118896484 + - 0.017715454 + - -0.52490234 + - -0.16210938 + - 0.7363281 + - -0.078552246 + - 0.5415039 + - 0.28173828 + - 0.2770996 + diff --git a/backends/candle/tests/snapshots/test_flash_bert__mini_single.snap b/backends/candle/tests/snapshots/test_flash_bert__mini_single.snap new file mode 100644 index 00000000..87eb16c9 --- /dev/null +++ b/backends/candle/tests/snapshots/test_flash_bert__mini_single.snap @@ -0,0 +1,389 @@ +--- +source: backends/candle/tests/test_flash_bert.rs +expression: embeddings_single +--- +- - -0.58984375 + - -0.25170898 + - 0.14880371 + - 0.055023193 + - -0.11743164 + - -0.1665039 + - -0.23291016 + - -0.07183838 + - -0.19958496 + - -0.2956543 + - -0.7734375 + - 0.1439209 + - -0.29077148 + - -0.18530273 + - -0.9091797 + - -0.020324707 + - 0.02810669 + - 0.34814453 + - -1.0029297 + - -0.4165039 + - 0.19360352 + - 0.40112305 + - -0.39111328 + - -0.18859863 + - 0.45239258 + - 0.2565918 + - 0.17712402 + - -0.31347656 + - 0.12524414 + - -0.08306885 + - 0.64941406 + - -0.23059082 + - 0.18432617 + - 0.49536133 + - -0.36572266 + - 0.83691406 + - -0.55078125 + - 0.10412598 + - 0.09442139 + - -0.21032715 + - -0.29418945 + - -0.1887207 + - -0.009468079 + - 0.18676758 + - 0.92578125 + - 0.33911133 + - -0.08227539 + - -0.60253906 + - 0.15283203 + - 0.2097168 + - -0.20947266 + - -0.13903809 + - -0.38330078 + - 0.6376953 + - 0.0038871765 + - 0.09881592 + - 0.2783203 + - -0.022033691 + - -0.5361328 + - 0.3203125 + - 0.48510742 + - -0.69384766 + - 0.02470398 + - 0.23547363 + - 0.29467773 + - -0.2705078 + - 0.04840088 + - 0.1104126 + - 0.5395508 + - -0.7709961 + - 0.42089844 + - 0.4741211 + - -0.053131104 + - 0.07867432 + - 0.14379883 + - -0.43798828 + - 0.21801758 + - 0.055358887 + - 0.74609375 + - -0.21813965 + - 0.7265625 + - 0.5493164 + - -0.115478516 + - 0.36816406 + - 1.0117188 + - -0.51660156 + - 0.11773682 + - 0.06976318 + - -0.6767578 + - -0.5234375 + - -0.27124023 + - -0.52490234 + - -0.59472656 + - -0.0008368492 + - -0.29223633 + - -0.095581055 + - -0.052246094 + - -0.9140625 + - -0.4033203 + - 0.38378906 + - -0.022644043 + - 0.0635376 + - 0.41674805 + - -0.046722412 + - -0.06915283 + - 0.09020996 + - 0.42651367 + - 0.10870361 + - 0.31054688 + - -0.62353516 + - -0.19165039 + - 0.29467773 + - 0.18371582 + - -0.18945313 + - 0.16491699 + - -0.19360352 + - -0.10253906 + - -0.041015625 + - -0.14904785 + - 0.69140625 + - -0.87353516 + - -0.05947876 + - -0.2454834 + - -0.1282959 + - -0.29174805 + - -0.18945313 + - -0.3671875 + - 0 + - -0.0036792755 + - -0.36645508 + - -0.453125 + - 0.049804688 + - 0.3190918 + - -0.4267578 + - -0.20288086 + - 0.09411621 + - -0.42016602 + - 0.44628906 + - -0.14099121 + - 0.08117676 + - -0.13842773 + - 1.1357422 + - 0.3474121 + - 0.09552002 + - -0.64404297 + - 0.5019531 + - 0.8339844 + - -0.59472656 + - -0.061309814 + - 0.31933594 + - -0.17578125 + - 0.022354126 + - -0.054382324 + - -0.10699463 + - 0.15539551 + - 0.1661377 + - 0.21582031 + - -0.028503418 + - -0.5620117 + - 0.07647705 + - -0.16394043 + - -0.06903076 + - 0.3647461 + - 0.042266846 + - 0.01777649 + - -0.06201172 + - 0.2998047 + - -0.26342773 + - 0.34545898 + - 0.061767578 + - -0.062316895 + - -0.43237305 + - -0.2064209 + - -0.16894531 + - 0.20776367 + - -0.7114258 + - -0.6567383 + - -0.38549805 + - -0.053741455 + - -0.32617188 + - -0.18359375 + - -0.46020508 + - 0.29223633 + - 0.27734375 + - 0.21325684 + - 0.20617676 + - 0.12371826 + - 0.1817627 + - 0.5332031 + - 0.39941406 + - -0.46801758 + - 0.66503906 + - -0.09667969 + - 0.1616211 + - 0.6645508 + - 0.6796875 + - 1.046875 + - -0.09423828 + - 0.029663086 + - 0.097473145 + - 0.20336914 + - -0.57373047 + - -0.076660156 + - 0.08081055 + - 0.15478516 + - -0.26098633 + - -0.2565918 + - 0.7861328 + - -0.46923828 + - 0.30908203 + - -0.25512695 + - 0.0060653687 + - -0.52978516 + - 0.6074219 + - -0.24658203 + - -0.3017578 + - 0.26123047 + - 0.014434814 + - -0.7026367 + - -0.049987793 + - -0.14868164 + - -0.080200195 + - -0.16467285 + - 0 + - -0.25073242 + - 0.43286133 + - -0.87597656 + - 0.71728516 + - 0.29003906 + - 0.00737381 + - -0.625 + - 0.35473633 + - -0.5522461 + - 0.18908691 + - 0.203125 + - -0.035858154 + - 0.45239258 + - 0.109191895 + - 0.19152832 + - -0.16931152 + - -0.35546875 + - -0.3486328 + - -0.47021484 + - -0.2692871 + - 0.23547363 + - 0.5883789 + - -0.33862305 + - -0.24633789 + - -0.15942383 + - -0.22460938 + - -0.2548828 + - 0.12573242 + - -0.10620117 + - 0.2854004 + - 0.17150879 + - -0.32861328 + - -0.67822266 + - 0.44799805 + - 0.29223633 + - 0.5073242 + - 0.59228516 + - -0.28442383 + - -0.2626953 + - -0.4230957 + - 0.2175293 + - -0.009521484 + - 0.18823242 + - 0.25463867 + - -0.3190918 + - -0.05871582 + - 0.0067634583 + - -0.043914795 + - 0.3269043 + - -0.1965332 + - 0.3112793 + - 0.02748108 + - 0.03387451 + - -0.35595703 + - -0.25878906 + - -0.077819824 + - -0.109069824 + - 0.18725586 + - 0.14453125 + - 0.4350586 + - -0.18066406 + - -0.49487305 + - -0.11633301 + - -0.18017578 + - -0.16992188 + - 0.8046875 + - -0.5957031 + - 0.62890625 + - 0.03213501 + - 0.14562988 + - 0.80859375 + - 0.10900879 + - 0.3479004 + - 0.58154297 + - -0.54003906 + - -0.54345703 + - 0.056640625 + - -0.095458984 + - -0.33496094 + - -0.18286133 + - 0.39746094 + - -0.80908203 + - -0.10333252 + - 0.65283203 + - -0.090026855 + - 0.103881836 + - -0.24633789 + - 0.5131836 + - -0.35888672 + - -0.026290894 + - -0.24121094 + - 0.10424805 + - -0.6333008 + - 0.0016736984 + - -0.52734375 + - -0.000000059604645 + - -0.19885254 + - 0.08843994 + - 0.55566406 + - -0.7998047 + - 0.3083496 + - -0.18652344 + - 0.10205078 + - 0.8203125 + - 0.024368286 + - 0.07745361 + - 0.6020508 + - 0.01398468 + - -0.5292969 + - -0.15368652 + - 0.027145386 + - 0.6113281 + - 0.45922852 + - -0.0009241104 + - 0.41479492 + - 0.02420044 + - 0.87109375 + - 0.32104492 + - 0.12023926 + - -0.055725098 + - 0.22009277 + - -0.6298828 + - 0.36914063 + - 0.68066406 + - 0.105773926 + - -0.42504883 + - -0.2019043 + - 0.5336914 + - -0.12414551 + - -0.16540527 + - 0.019561768 + - 0.8852539 + - 0.30737305 + - -0.28955078 + - -0.058746338 + - -0.40527344 + - -0.4038086 + - 1.1376953 + - 0.27807617 + - -0.06793213 + - -0.46801758 + - 0.09051514 + - 0.3161621 + - -0.053741455 + - 0.14733887 + - 0.1171875 + - -0.1616211 + - 0.5151367 + - 0.62060547 + - 0.64160156 + - -0.036224365 + - 0.118896484 + - 0.017318726 + - -0.5253906 + - -0.16259766 + - 0.7363281 + - -0.078430176 + - 0.5415039 + - 0.28295898 + - 0.2763672 + diff --git a/backends/candle/tests/snapshots/test_jina__jina_batch.snap b/backends/candle/tests/snapshots/test_jina__jina_batch.snap new file mode 100644 index 00000000..545f5d25 --- /dev/null +++ b/backends/candle/tests/snapshots/test_jina__jina_batch.snap @@ -0,0 +1,1541 @@ +--- +source: backends/candle/tests/test_jina.rs +expression: embeddings_batch +--- +- - -0.73668385 + - -0.34012684 + - -0.0715523 + - 0.10852202 + - 0.14204022 + - -0.40989703 + - 0.23236847 + - 0.14997244 + - 0.22580299 + - -0.026758675 + - 0.39548117 + - -0.25018567 + - -0.61580956 + - -1.0374275 + - 0.3620463 + - -0.39094338 + - 0.30281666 + - -0.63823456 + - -0.06514027 + - -0.43619013 + - -0.56252056 + - -0.032389965 + - 0.7796189 + - -0.07337697 + - 0.6910109 + - -0.42153034 + - -0.44665217 + - -0.08850418 + - 0.9865168 + - 0.719132 + - -0.26575097 + - 0.348945 + - 0.2953208 + - 0.06414049 + - -0.6311697 + - 0.49613383 + - 0.5473903 + - 0.34628963 + - -0.30335656 + - -0.37656382 + - -0.24137187 + - 0.22599705 + - 0.7282061 + - 0.31632474 + - -0.5788445 + - -0.048453815 + - -0.99214715 + - 0.67147404 + - 0.40648526 + - -0.08697222 + - -0.09709507 + - 0.13716747 + - -0.3302538 + - 0.6385713 + - -0.4333674 + - 0.17140126 + - -0.10378386 + - 0.34555072 + - -0.35002294 + - 0.37138405 + - 0.6942465 + - -0.012582751 + - -0.29724523 + - 0.3630598 + - -0.3450033 + - -0.58791435 + - -0.2560966 + - -0.06847818 + - -0.82814556 + - 0.5919305 + - -0.5577107 + - -0.3965461 + - 0.3292896 + - -0.66765755 + - 0.10509022 + - -0.029757977 + - 0.18136811 + - 0.74675995 + - -0.2849062 + - -0.7784263 + - -0.46817058 + - -0.15993918 + - -0.5867964 + - 0.81102645 + - 0.40201172 + - -0.8813075 + - 0.13769837 + - 0.21844776 + - 0.65046084 + - -0.4814538 + - 0.13929038 + - 0.1781513 + - -0.25574976 + - -0.6823837 + - -0.32714817 + - -0.7217614 + - 0.672464 + - 0.27874002 + - -0.8074308 + - 0.7273515 + - 0.6546895 + - 0.3032117 + - 0.3657865 + - -0.19880594 + - -0.07629948 + - 0.4256604 + - -1.078911 + - -0.20131199 + - -1.1999272 + - -0.59212506 + - -0.17580846 + - -0.3750479 + - -0.41822523 + - -0.2924359 + - -0.37015983 + - -0.15523282 + - 0.61704636 + - -0.6305304 + - 0.044137325 + - 0.11221082 + - 1.0062258 + - -0.0049912357 + - -0.47516233 + - 0.56906956 + - 0.3740537 + - 0.8261896 + - -0.32491022 + - -0.005009859 + - -0.23551618 + - -0.039758164 + - -0.12311789 + - 0.71079177 + - -0.36461496 + - 0.6814915 + - 0.8928003 + - 0.07970085 + - 0.30849716 + - 0.2967249 + - -0.2721908 + - 0.75819844 + - -0.29332206 + - -0.5996169 + - 0.56722975 + - -0.379029 + - -0.24413143 + - 0.19070697 + - 1.0095695 + - -0.40070263 + - -0.48254392 + - 0.27729446 + - -0.27582094 + - 0.46122116 + - -0.8984841 + - -0.61792296 + - -0.51835155 + - 0.8659856 + - -0.063467644 + - 0.17027345 + - 0.27787298 + - -0.52405035 + - -0.4709656 + - 0.74265236 + - -0.9647818 + - 0.34670112 + - -0.03367323 + - -0.29435757 + - 0.7839863 + - 0.954704 + - -0.074295476 + - -0.11577297 + - 0.34262037 + - 0.60408914 + - -0.15067175 + - -0.56553507 + - -0.45733213 + - -0.2838439 + - -0.562523 + - 0.4098379 + - -0.15422437 + - -0.26893747 + - 0.063611485 + - -0.72149956 + - -0.101162024 + - 0.11956217 + - 0.19708382 + - -0.3307448 + - -0.15375762 + - -0.30214736 + - -0.25265673 + - -0.49330828 + - -0.3747701 + - 0.011120404 + - 0.18189253 + - -0.030484946 + - -0.6194291 + - 0.21167146 + - 0.227614 + - 0.16227365 + - -0.08826481 + - 0.49216056 + - 0.2937846 + - 0.8931769 + - -0.59395313 + - 0.74639904 + - 0.00653029 + - 0.34859207 + - -0.6235556 + - -0.4537472 + - -0.63391864 + - -0.1443559 + - 0.07003089 + - 0.037237544 + - 0.9805306 + - 0.2809817 + - 0.073027 + - 0.32501057 + - -0.37974665 + - -0.4622294 + - 0.39487654 + - 0.06744876 + - 0.31691965 + - -0.049856696 + - 0.13301113 + - 0.28951162 + - 0.03810055 + - -0.38808325 + - 0.043499887 + - 0.008641668 + - -0.24204402 + - -0.090396896 + - 0.18011324 + - 0.23784415 + - -0.5009206 + - 0.1553507 + - 0.5209598 + - 0.3395558 + - -0.5927804 + - -0.025065303 + - 0.15375154 + - 0.077146195 + - 0.16486229 + - 0.9559272 + - 0.34631437 + - -0.455141 + - -0.39787167 + - 0.29098955 + - 0.5584746 + - 0.059295062 + - 0.1384459 + - 0.42039558 + - 0.010655655 + - 0.021553522 + - -0.4591532 + - -0.69064206 + - -0.0022638962 + - -0.030271845 + - 0.25798056 + - 0.7935306 + - -0.25783426 + - -0.14395986 + - 0.2626888 + - 0.28115913 + - -0.34353155 + - 1.0121855 + - -0.2966 + - 0.3429189 + - -0.5701311 + - -0.09280105 + - 0.48118117 + - 0.35712275 + - -0.16092014 + - 0.116320096 + - -0.16216387 + - -0.4784748 + - -0.49545664 + - 0.026475558 + - 0.58664286 + - -0.20623466 + - 0.52622694 + - -0.26142675 + - -0.82888204 + - -0.05105743 + - -0.13397925 + - -0.45493487 + - 0.45166442 + - 0.28790733 + - 0.9281663 + - 0.5534405 + - 0.24245916 + - 0.24755414 + - 0.47102252 + - -0.22941728 + - -0.059087906 + - -0.12679414 + - -0.23186122 + - 0.3594575 + - 0.36945596 + - 0.38604233 + - 0.04756975 + - 0.09457792 + - 0.7681518 + - 0.51292485 + - -0.4647774 + - -0.41134024 + - -0.14788796 + - -0.026648177 + - 0.13731936 + - 0.1844856 + - -0.4147409 + - -0.4478915 + - -0.027359122 + - -0.3959928 + - -0.37414354 + - -0.19002788 + - 0.39353797 + - 0.007390729 + - 0.37485725 + - -0.12709224 + - -0.46179503 + - -0.14690821 + - -0.06336937 + - 0.037377708 + - 0.07596908 + - -0.34052616 + - 0.46046728 + - 0.5605055 + - -0.55663145 + - -0.35354254 + - 0.06384647 + - -0.14710614 + - -0.22530709 + - 0.52121586 + - -0.0495451 + - -0.8934284 + - -0.111877166 + - 0.31237033 + - -0.6044664 + - -0.7920669 + - 0.017268227 + - -0.31899163 + - 0.10419905 + - -0.34517533 + - -0.3948041 + - 0.5101954 + - -0.08465504 + - 0.25743407 + - 0.5437978 + - -0.6793512 + - -0.45921823 + - 0.5326503 + - -0.18141039 + - -0.54093796 + - 0.039407533 + - 0.017359953 + - -1.1775223 + - 0.10029836 + - 0.17366055 + - -0.85059124 + - -0.2780114 + - 0.96832466 + - -0.6595807 + - -0.7359334 + - 0.24694958 + - 0.7053874 + - 0.07484163 + - -0.06028542 + - -0.2428803 + - 0.026168117 + - 0.32321474 + - 0.6359962 + - 0.6015481 + - -0.23883562 + - -0.26510406 + - -1.3345877 + - -0.3764 + - 0.07601651 + - 0.27453107 + - -0.3821561 + - -0.23207092 + - -0.09694352 + - -0.4056513 + - 0.6571864 + - 0.13454896 + - -0.4461467 + - -0.30312943 + - 0.1957128 + - 0.015960306 + - 0.40737882 + - 0.36711743 + - 0.07948569 + - -0.3717199 + - 0.16874814 + - -0.31267354 + - -0.23825826 + - -0.0774443 + - -0.38939747 + - -0.14293732 + - 0.90112275 + - -0.72516924 + - 0.984644 + - 0.019674359 + - 0.055611532 + - -0.2934082 + - 0.46005297 + - -0.19833238 + - -0.45715576 + - -0.3313622 + - -0.027743893 + - 0.38861424 + - 0.15984824 + - 0.44063833 + - 0.30860153 + - 0.27471563 + - -0.13229555 + - -0.51096684 + - -0.8010029 + - -0.4840509 + - -0.519855 + - 0.21440867 + - 0.06478418 + - 0.6136018 + - 0.05823111 + - 0.02230786 + - -0.31735128 + - 0.45651013 + - 0.05455484 + - 0.03172924 + - -0.15754183 + - 0.57087106 + - 0.302311 + - 0.32235426 + - -0.37271267 + - -0.062497634 + - 0.49394235 + - -0.5000919 + - 0.47894144 + - -0.055776197 + - -0.7185101 + - 0.34229258 + - -0.47906867 + - 1.2254735 + - -0.63414246 + - 0.091453776 + - 0.04621964 + - 0.39976168 + - 1.1370289 + - 0.8316378 + - -0.1304873 + - -0.19904993 + - 0.366474 + - 0.40303475 + - 0.19891736 + - -0.2694296 + - -0.007872573 + - -0.21664867 + - 0.07184189 + - -0.30892542 + - -0.51357216 + - -0.5767418 + - 0.6941703 + - -0.054563563 + - -0.5326759 + - -0.47848055 + - 0.40909168 + - 0.11241865 + - 0.43552828 + - 0.08060895 + - -0.3370398 + - 0.17169443 + - 0.21128477 + - -0.65164536 + - -1.0550821 + - -0.3331693 + - 0.5605589 + - 0.8862934 + - -0.022461575 + - -0.13918051 + - 0.85254467 + - -0.26736432 + - 0.40900162 + - -0.54530114 + - -0.109466515 + - -0.5180425 + - -1.1476005 + - 0.017990198 + - 0.22895983 + - -0.48230448 + - 0.32160607 + - 0.71078336 + - 0.48590115 + - -0.64358234 + - -0.75938046 + - 0.105871856 + - -0.4471452 + - 0.071705274 + - 0.46669033 + - 0.29078048 + - -0.109523356 + - 0.56042516 + - 0.11813028 + - 0.46236435 + - -0.7612805 + - 0.08246316 + - 0.561123 + - -0.22669752 + - 0.46435124 + - -0.12556691 + - -0.17807259 + - -0.38081747 + - 0.089175105 + - 0.37251982 + - 0.009899339 +- - -0.6258575 + - -0.54804754 + - 0.13834608 + - 0.3368268 + - 0.118097365 + - -0.44837838 + - 0.38703012 + - -0.03681364 + - 0.16565804 + - -0.17456621 + - 0.46841913 + - -0.16639332 + - -0.6830714 + - -0.9968887 + - 0.74894214 + - -0.49739882 + - 0.17090584 + - -0.51595217 + - -0.025067393 + - -0.43994397 + - -0.5288875 + - -0.010524412 + - 0.68397003 + - 0.00094923377 + - 0.7182251 + - -0.7732404 + - -0.53631896 + - -0.014902936 + - 0.8814053 + - 0.49463782 + - -0.3484184 + - 0.2203561 + - 0.535584 + - 0.053611778 + - -0.6109779 + - 0.34571558 + - 0.55962723 + - 0.42110342 + - -0.3431571 + - -0.42719498 + - -0.1622013 + - -0.063774794 + - 0.7308566 + - 0.37309992 + - -0.54693377 + - 0.10378383 + - -0.82453156 + - 0.56506014 + - 0.60045683 + - -0.012032587 + - -0.17478134 + - 0.24510865 + - -0.5098945 + - 0.9484012 + - -0.29771543 + - 0.044558473 + - -0.2106627 + - 0.31117538 + - -0.60099584 + - 0.18622015 + - 0.693832 + - 0.09960179 + - -0.42590287 + - 0.32845956 + - -0.40717852 + - -0.44070238 + - 0.028581027 + - -0.19334748 + - -0.8829633 + - 0.4422871 + - -0.7548492 + - -0.39266592 + - 0.69387734 + - -0.6330214 + - 0.21701033 + - 0.18680653 + - 0.09905142 + - 0.8045387 + - -0.315876 + - -0.7137362 + - -0.5562205 + - -0.1079926 + - -0.4724413 + - 0.96796775 + - 0.29352766 + - -0.81541336 + - 0.36886823 + - 0.4787166 + - 0.52278715 + - -0.48305058 + - 0.13325576 + - 0.4875254 + - -0.22803363 + - -0.7590358 + - -0.14664412 + - -0.879706 + - 0.682916 + - 0.14855725 + - -1.0184903 + - 0.5507337 + - 0.569222 + - 0.31514075 + - 0.62274027 + - -0.32432622 + - -0.09131627 + - 0.43202394 + - -0.8701954 + - -0.08341806 + - -1.2153687 + - -0.40948495 + - -0.1702378 + - -0.54740334 + - -0.48194543 + - -0.2423253 + - -0.4687544 + - -0.13046497 + - 0.66072136 + - -0.514049 + - -0.048983276 + - -0.1139099 + - 1.1299253 + - -0.06445108 + - -0.26474863 + - 0.672685 + - 0.28219154 + - 0.83556813 + - -0.3372388 + - -0.010138115 + - -0.30575222 + - 0.12501504 + - -0.18299821 + - 0.68838155 + - -0.20157899 + - 0.66440886 + - 0.98802114 + - 0.07312933 + - 0.37954614 + - 0.276943 + - -0.2548735 + - 0.9237343 + - -0.3586478 + - -0.65259564 + - 0.67842615 + - -0.47164488 + - -0.20399868 + - 0.09143736 + - 0.84724796 + - -0.47076935 + - -0.24657816 + - 0.157211 + - -0.1467024 + - 0.42150247 + - -0.9810306 + - -0.29296878 + - -0.6768669 + - 0.7368343 + - -0.016836716 + - 0.38332418 + - 0.26103643 + - -0.43472654 + - -0.42027396 + - 0.7256403 + - -0.7409004 + - 0.2097269 + - -0.14758325 + - -0.32247585 + - 0.768879 + - 1.0538769 + - -0.13251239 + - -0.15693328 + - 0.30749947 + - 0.66435075 + - -0.35877824 + - -0.63255584 + - -0.7470684 + - -0.39004886 + - -0.717204 + - 0.1962583 + - -0.31756508 + - -0.14924219 + - -0.048519254 + - -0.64350164 + - -0.08571612 + - 0.16110295 + - 0.2818928 + - -0.1855998 + - -0.013100313 + - -0.32920322 + - -0.08167001 + - -0.60771966 + - -0.16713963 + - -0.042582974 + - 0.2164097 + - -0.055456955 + - -0.83404255 + - 0.15130463 + - 0.23197632 + - 0.24511193 + - -0.36819267 + - 0.46589231 + - 0.1815627 + - 1.1352087 + - -0.6556176 + - 0.61636865 + - -0.048470926 + - 0.6031747 + - -0.43357193 + - -0.3559264 + - -0.57808846 + - -0.20370248 + - 0.2139466 + - 0.17453064 + - 0.9311236 + - 0.103386104 + - 0.06571435 + - 0.5040777 + - -0.48878047 + - -0.629567 + - 0.5088647 + - -0.009554142 + - 0.31415233 + - -0.13485786 + - -0.06238198 + - 0.2109054 + - -0.07454253 + - -0.5029938 + - -0.03624041 + - -0.15522996 + - -0.44486445 + - -0.12613204 + - 0.3527791 + - 0.01934108 + - -0.6815807 + - 0.15968806 + - 0.40955836 + - 0.37752843 + - -0.60240865 + - 0.06797554 + - 0.13444227 + - 0.20517793 + - 0.06795175 + - 0.72272956 + - 0.40924573 + - -0.3583372 + - -0.24953146 + - 0.3454433 + - 0.73249006 + - -0.016263098 + - 0.38937235 + - 0.477409 + - -0.24734643 + - -0.22065666 + - -0.43827358 + - -0.8564954 + - -0.16533723 + - 0.073783174 + - 0.21324871 + - 0.94114935 + - -0.4757835 + - -0.15708917 + - 0.18403076 + - 0.1925097 + - -0.21202324 + - 0.9842098 + - -0.0878644 + - 0.2447358 + - -0.94101125 + - 0.087468654 + - 0.6926329 + - 0.46830428 + - -0.29974294 + - -0.08031401 + - 0.03146644 + - -0.40275338 + - -0.5026202 + - 0.029807257 + - 0.8209338 + - -0.026684912 + - 0.3648769 + - -0.21028933 + - -0.7029544 + - 0.06057348 + - 0.019458666 + - -0.5838164 + - 0.4917751 + - 0.3183142 + - 0.84544075 + - 0.44492447 + - 0.21333058 + - 0.30247962 + - 0.4446727 + - -0.4675721 + - -0.060020737 + - -0.3299496 + - -0.11601128 + - 0.35059682 + - 0.3092138 + - 0.35126644 + - -0.31546658 + - -0.0014095213 + - 0.59492683 + - 0.6572098 + - -0.6723788 + - -0.43647325 + - 0.013674368 + - -0.29085657 + - 0.31547442 + - 0.11456098 + - -0.24219348 + - -0.4402923 + - -0.020581847 + - -0.39323604 + - -0.22420351 + - -0.09342688 + - 0.38753325 + - -0.007150747 + - 0.6456237 + - -0.066635594 + - -0.7063716 + - -0.29063946 + - -0.19057932 + - 0.20498846 + - 0.039399635 + - -0.27865243 + - 0.5537731 + - 0.73458505 + - -0.6050471 + - -0.34599435 + - -0.1824834 + - -0.058190476 + - -0.21754293 + - 0.5583502 + - -0.060159765 + - -0.81757104 + - -0.2916988 + - 0.26248676 + - -0.7621182 + - -1.0135679 + - -0.051746085 + - -0.4234231 + - 0.019949561 + - -0.3188232 + - -0.3598222 + - 0.6737609 + - 0.025792923 + - 0.24157763 + - 0.66929823 + - -0.57916325 + - -0.48869452 + - 0.50248295 + - -0.22694115 + - -0.37696207 + - -0.10308571 + - -0.03803464 + - -1.2336473 + - 0.09218389 + - -0.024684472 + - -0.7819303 + - -0.34423468 + - 1.0444496 + - -0.5913346 + - -0.5812771 + - 0.32361537 + - 0.50330496 + - 0.20229712 + - -0.33572042 + - -0.099175066 + - -0.2046279 + - 0.40374172 + - 0.46500006 + - 0.7372935 + - -0.1699504 + - -0.39697903 + - -1.0821605 + - -0.33508205 + - 0.040522095 + - 0.21879213 + - -0.5729624 + - -0.27783832 + - -0.17614284 + - -0.28024828 + - 0.7211515 + - -0.00059991144 + - -0.45371056 + - -0.28531542 + - 0.14602807 + - 0.04662592 + - 0.391477 + - 0.58497566 + - 0.010007991 + - -0.35190052 + - 0.039734583 + - -0.27348435 + - -0.07543832 + - 0.11173277 + - -0.27350292 + - -0.092721514 + - 0.945073 + - -0.77325374 + - 0.7610948 + - -0.13234852 + - 0.12271928 + - -0.09632763 + - 0.08469461 + - -0.23362634 + - -0.5520911 + - -0.5005254 + - -0.093050085 + - 0.44351986 + - -0.05776892 + - 0.53914416 + - 0.37950468 + - 0.3350498 + - -0.04806131 + - -0.20750779 + - -0.8747888 + - -0.50522244 + - -0.26718557 + - 0.15583205 + - -0.14595877 + - 0.62023103 + - -0.06198295 + - -0.14184205 + - -0.3325085 + - 0.43675852 + - -0.023822654 + - 0.0666159 + - -0.41536772 + - 0.59659994 + - 0.34701145 + - 0.43234462 + - -0.16687086 + - -0.2053485 + - 0.61458313 + - -0.5899166 + - 0.34109336 + - 0.11660128 + - -0.97081697 + - 0.24870658 + - -0.19222844 + - 1.2454556 + - -0.7412164 + - 0.040932313 + - 0.09928498 + - 0.43134254 + - 1.3612144 + - 0.9518776 + - -0.14346507 + - -0.124920174 + - 0.27368224 + - 0.2799998 + - 0.30024117 + - -0.27684867 + - 0.054528676 + - -0.23937213 + - 0.01023684 + - -0.47547257 + - -0.6867144 + - -0.3181229 + - 0.52634346 + - 0.024729064 + - -0.47309282 + - -0.3616146 + - 0.17723028 + - 0.018263185 + - 0.36472362 + - 0.422522 + - -0.22185251 + - 0.112825364 + - 0.23129265 + - -0.68728304 + - -1.1391162 + - -0.49052075 + - 0.49895275 + - 1.1125307 + - 0.13734463 + - -0.01637707 + - 0.7318877 + - -0.3134311 + - 0.57362086 + - -0.46828306 + - -0.19481543 + - -0.69564104 + - -1.0398626 + - -0.20918602 + - -0.03195838 + - -0.4123183 + - 0.39416733 + - 0.82587814 + - 0.4731025 + - -0.48742402 + - -0.7621099 + - 0.0016704369 + - -0.49811795 + - 0.18096921 + - 0.5582192 + - 0.47097775 + - -0.20861416 + - 0.5764632 + - -0.12298558 + - 0.49796546 + - -0.7445811 + - 0.10852885 + - 0.6405106 + - -0.08042282 + - 0.5474277 + - 0.029673172 + - -0.17238168 + - -0.3215491 + - 0.19171812 + - 0.459572 + - 0.16634196 +- - -0.73668385 + - -0.34012684 + - -0.0715523 + - 0.10852202 + - 0.14204022 + - -0.40989703 + - 0.23236847 + - 0.14997244 + - 0.22580299 + - -0.026758675 + - 0.39548117 + - -0.25018567 + - -0.61580956 + - -1.0374275 + - 0.3620463 + - -0.39094338 + - 0.30281666 + - -0.63823456 + - -0.06514027 + - -0.43619013 + - -0.56252056 + - -0.032389965 + - 0.7796189 + - -0.07337697 + - 0.6910109 + - -0.42153034 + - -0.44665217 + - -0.08850418 + - 0.9865168 + - 0.719132 + - -0.26575097 + - 0.348945 + - 0.2953208 + - 0.06414049 + - -0.6311697 + - 0.49613383 + - 0.5473903 + - 0.34628963 + - -0.30335656 + - -0.37656382 + - -0.24137187 + - 0.22599705 + - 0.7282061 + - 0.31632474 + - -0.5788445 + - -0.048453815 + - -0.99214715 + - 0.67147404 + - 0.40648526 + - -0.08697222 + - -0.09709507 + - 0.13716747 + - -0.3302538 + - 0.6385713 + - -0.4333674 + - 0.17140126 + - -0.10378386 + - 0.34555072 + - -0.35002294 + - 0.37138405 + - 0.6942465 + - -0.012582751 + - -0.29724523 + - 0.3630598 + - -0.3450033 + - -0.58791435 + - -0.2560966 + - -0.06847818 + - -0.82814556 + - 0.5919305 + - -0.5577107 + - -0.3965461 + - 0.3292896 + - -0.66765755 + - 0.10509022 + - -0.029757977 + - 0.18136811 + - 0.74675995 + - -0.2849062 + - -0.7784263 + - -0.46817058 + - -0.15993918 + - -0.5867964 + - 0.81102645 + - 0.40201172 + - -0.8813075 + - 0.13769837 + - 0.21844776 + - 0.65046084 + - -0.4814538 + - 0.13929038 + - 0.1781513 + - -0.25574976 + - -0.6823837 + - -0.32714817 + - -0.7217614 + - 0.672464 + - 0.27874002 + - -0.8074308 + - 0.7273515 + - 0.6546895 + - 0.3032117 + - 0.3657865 + - -0.19880594 + - -0.07629948 + - 0.4256604 + - -1.078911 + - -0.20131199 + - -1.1999272 + - -0.59212506 + - -0.17580846 + - -0.3750479 + - -0.41822523 + - -0.2924359 + - -0.37015983 + - -0.15523282 + - 0.61704636 + - -0.6305304 + - 0.044137325 + - 0.11221082 + - 1.0062258 + - -0.0049912357 + - -0.47516233 + - 0.56906956 + - 0.3740537 + - 0.8261896 + - -0.32491022 + - -0.005009859 + - -0.23551618 + - -0.039758164 + - -0.12311789 + - 0.71079177 + - -0.36461496 + - 0.6814915 + - 0.8928003 + - 0.07970085 + - 0.30849716 + - 0.2967249 + - -0.2721908 + - 0.75819844 + - -0.29332206 + - -0.5996169 + - 0.56722975 + - -0.379029 + - -0.24413143 + - 0.19070697 + - 1.0095695 + - -0.40070263 + - -0.48254392 + - 0.27729446 + - -0.27582094 + - 0.46122116 + - -0.8984841 + - -0.61792296 + - -0.51835155 + - 0.8659856 + - -0.063467644 + - 0.17027345 + - 0.27787298 + - -0.52405035 + - -0.4709656 + - 0.74265236 + - -0.9647818 + - 0.34670112 + - -0.03367323 + - -0.29435757 + - 0.7839863 + - 0.954704 + - -0.074295476 + - -0.11577297 + - 0.34262037 + - 0.60408914 + - -0.15067175 + - -0.56553507 + - -0.45733213 + - -0.2838439 + - -0.562523 + - 0.4098379 + - -0.15422437 + - -0.26893747 + - 0.063611485 + - -0.72149956 + - -0.101162024 + - 0.11956217 + - 0.19708382 + - -0.3307448 + - -0.15375762 + - -0.30214736 + - -0.25265673 + - -0.49330828 + - -0.3747701 + - 0.011120404 + - 0.18189253 + - -0.030484946 + - -0.6194291 + - 0.21167146 + - 0.227614 + - 0.16227365 + - -0.08826481 + - 0.49216056 + - 0.2937846 + - 0.8931769 + - -0.59395313 + - 0.74639904 + - 0.00653029 + - 0.34859207 + - -0.6235556 + - -0.4537472 + - -0.63391864 + - -0.1443559 + - 0.07003089 + - 0.037237544 + - 0.9805306 + - 0.2809817 + - 0.073027 + - 0.32501057 + - -0.37974665 + - -0.4622294 + - 0.39487654 + - 0.06744876 + - 0.31691965 + - -0.049856696 + - 0.13301113 + - 0.28951162 + - 0.03810055 + - -0.38808325 + - 0.043499887 + - 0.008641668 + - -0.24204402 + - -0.090396896 + - 0.18011324 + - 0.23784415 + - -0.5009206 + - 0.1553507 + - 0.5209598 + - 0.3395558 + - -0.5927804 + - -0.025065303 + - 0.15375154 + - 0.077146195 + - 0.16486229 + - 0.9559272 + - 0.34631437 + - -0.455141 + - -0.39787167 + - 0.29098955 + - 0.5584746 + - 0.059295062 + - 0.1384459 + - 0.42039558 + - 0.010655655 + - 0.021553522 + - -0.4591532 + - -0.69064206 + - -0.0022638962 + - -0.030271845 + - 0.25798056 + - 0.7935306 + - -0.25783426 + - -0.14395986 + - 0.2626888 + - 0.28115913 + - -0.34353155 + - 1.0121855 + - -0.2966 + - 0.3429189 + - -0.5701311 + - -0.09280105 + - 0.48118117 + - 0.35712275 + - -0.16092014 + - 0.116320096 + - -0.16216387 + - -0.4784748 + - -0.49545664 + - 0.026475558 + - 0.58664286 + - -0.20623466 + - 0.52622694 + - -0.26142675 + - -0.82888204 + - -0.05105743 + - -0.13397925 + - -0.45493487 + - 0.45166442 + - 0.28790733 + - 0.9281663 + - 0.5534405 + - 0.24245916 + - 0.24755414 + - 0.47102252 + - -0.22941728 + - -0.059087906 + - -0.12679414 + - -0.23186122 + - 0.3594575 + - 0.36945596 + - 0.38604233 + - 0.04756975 + - 0.09457792 + - 0.7681518 + - 0.51292485 + - -0.4647774 + - -0.41134024 + - -0.14788796 + - -0.026648177 + - 0.13731936 + - 0.1844856 + - -0.4147409 + - -0.4478915 + - -0.027359122 + - -0.3959928 + - -0.37414354 + - -0.19002788 + - 0.39353797 + - 0.007390729 + - 0.37485725 + - -0.12709224 + - -0.46179503 + - -0.14690821 + - -0.06336937 + - 0.037377708 + - 0.07596908 + - -0.34052616 + - 0.46046728 + - 0.5605055 + - -0.55663145 + - -0.35354254 + - 0.06384647 + - -0.14710614 + - -0.22530709 + - 0.52121586 + - -0.0495451 + - -0.8934284 + - -0.111877166 + - 0.31237033 + - -0.6044664 + - -0.7920669 + - 0.017268227 + - -0.31899163 + - 0.10419905 + - -0.34517533 + - -0.3948041 + - 0.5101954 + - -0.08465504 + - 0.25743407 + - 0.5437978 + - -0.6793512 + - -0.45921823 + - 0.5326503 + - -0.18141039 + - -0.54093796 + - 0.039407533 + - 0.017359953 + - -1.1775223 + - 0.10029836 + - 0.17366055 + - -0.85059124 + - -0.2780114 + - 0.96832466 + - -0.6595807 + - -0.7359334 + - 0.24694958 + - 0.7053874 + - 0.07484163 + - -0.06028542 + - -0.2428803 + - 0.026168117 + - 0.32321474 + - 0.6359962 + - 0.6015481 + - -0.23883562 + - -0.26510406 + - -1.3345877 + - -0.3764 + - 0.07601651 + - 0.27453107 + - -0.3821561 + - -0.23207092 + - -0.09694352 + - -0.4056513 + - 0.6571864 + - 0.13454896 + - -0.4461467 + - -0.30312943 + - 0.1957128 + - 0.015960306 + - 0.40737882 + - 0.36711743 + - 0.07948569 + - -0.3717199 + - 0.16874814 + - -0.31267354 + - -0.23825826 + - -0.0774443 + - -0.38939747 + - -0.14293732 + - 0.90112275 + - -0.72516924 + - 0.984644 + - 0.019674359 + - 0.055611532 + - -0.2934082 + - 0.46005297 + - -0.19833238 + - -0.45715576 + - -0.3313622 + - -0.027743893 + - 0.38861424 + - 0.15984824 + - 0.44063833 + - 0.30860153 + - 0.27471563 + - -0.13229555 + - -0.51096684 + - -0.8010029 + - -0.4840509 + - -0.519855 + - 0.21440867 + - 0.06478418 + - 0.6136018 + - 0.05823111 + - 0.02230786 + - -0.31735128 + - 0.45651013 + - 0.05455484 + - 0.03172924 + - -0.15754183 + - 0.57087106 + - 0.302311 + - 0.32235426 + - -0.37271267 + - -0.062497634 + - 0.49394235 + - -0.5000919 + - 0.47894144 + - -0.055776197 + - -0.7185101 + - 0.34229258 + - -0.47906867 + - 1.2254735 + - -0.63414246 + - 0.091453776 + - 0.04621964 + - 0.39976168 + - 1.1370289 + - 0.8316378 + - -0.1304873 + - -0.19904993 + - 0.366474 + - 0.40303475 + - 0.19891736 + - -0.2694296 + - -0.007872573 + - -0.21664867 + - 0.07184189 + - -0.30892542 + - -0.51357216 + - -0.5767418 + - 0.6941703 + - -0.054563563 + - -0.5326759 + - -0.47848055 + - 0.40909168 + - 0.11241865 + - 0.43552828 + - 0.08060895 + - -0.3370398 + - 0.17169443 + - 0.21128477 + - -0.65164536 + - -1.0550821 + - -0.3331693 + - 0.5605589 + - 0.8862934 + - -0.022461575 + - -0.13918051 + - 0.85254467 + - -0.26736432 + - 0.40900162 + - -0.54530114 + - -0.109466515 + - -0.5180425 + - -1.1476005 + - 0.017990198 + - 0.22895983 + - -0.48230448 + - 0.32160607 + - 0.71078336 + - 0.48590115 + - -0.64358234 + - -0.75938046 + - 0.105871856 + - -0.4471452 + - 0.071705274 + - 0.46669033 + - 0.29078048 + - -0.109523356 + - 0.56042516 + - 0.11813028 + - 0.46236435 + - -0.7612805 + - 0.08246316 + - 0.561123 + - -0.22669752 + - 0.46435124 + - -0.12556691 + - -0.17807259 + - -0.38081747 + - 0.089175105 + - 0.37251982 + - 0.009899339 + diff --git a/backends/candle/tests/snapshots/test_jina__jina_single.snap b/backends/candle/tests/snapshots/test_jina__jina_single.snap new file mode 100644 index 00000000..a36afe6a --- /dev/null +++ b/backends/candle/tests/snapshots/test_jina__jina_single.snap @@ -0,0 +1,517 @@ +--- +source: backends/candle/tests/test_jina.rs +expression: embeddings_single +--- +- - -0.7366036 + - -0.34022003 + - -0.0716392 + - 0.108484305 + - 0.14210966 + - -0.4099645 + - 0.23231864 + - 0.14997455 + - 0.22580643 + - -0.026713861 + - 0.3955075 + - -0.25014502 + - -0.6157121 + - -1.0374233 + - 0.3620551 + - -0.39090034 + - 0.3028331 + - -0.63823646 + - -0.06523182 + - -0.43618122 + - -0.56259745 + - -0.032301467 + - 0.7795808 + - -0.073509656 + - 0.6909338 + - -0.42143556 + - -0.44656378 + - -0.08844421 + - 0.98647565 + - 0.7190607 + - -0.26573643 + - 0.348924 + - 0.29520458 + - 0.06409048 + - -0.63120323 + - 0.49616978 + - 0.5474262 + - 0.34614715 + - -0.30342722 + - -0.37654096 + - -0.24154605 + - 0.22582027 + - 0.7281335 + - 0.3162906 + - -0.5788307 + - -0.048332658 + - -0.9921505 + - 0.671495 + - 0.40651703 + - -0.086957395 + - -0.096983805 + - 0.13722567 + - -0.3303092 + - 0.6385299 + - -0.43336847 + - 0.1714337 + - -0.10392817 + - 0.3456139 + - -0.35007638 + - 0.37135443 + - 0.69419295 + - -0.012638569 + - -0.29731104 + - 0.3629211 + - -0.34492856 + - -0.5877692 + - -0.25613576 + - -0.06834865 + - -0.8281671 + - 0.59186125 + - -0.55771023 + - -0.3964429 + - 0.3292411 + - -0.66763717 + - 0.1050239 + - -0.029753016 + - 0.18126683 + - 0.7467178 + - -0.28494564 + - -0.77837753 + - -0.46805096 + - -0.15983714 + - -0.5867695 + - 0.8109579 + - 0.40202174 + - -0.8813922 + - 0.13766704 + - 0.21849489 + - 0.65029484 + - -0.48126322 + - 0.13934065 + - 0.17813931 + - -0.2557544 + - -0.68231326 + - -0.32716078 + - -0.7217004 + - 0.6723558 + - 0.27870503 + - -0.80748856 + - 0.7273158 + - 0.6544879 + - 0.3032456 + - 0.36579895 + - -0.19886778 + - -0.07636473 + - 0.42568275 + - -1.0787318 + - -0.20126738 + - -1.1998513 + - -0.59218585 + - -0.17576711 + - -0.37495485 + - -0.41820025 + - -0.29240656 + - -0.37017444 + - -0.1553841 + - 0.61696583 + - -0.630497 + - 0.044032294 + - 0.11211239 + - 1.0062819 + - -0.004947958 + - -0.47510833 + - 0.56907827 + - 0.37397504 + - 0.826095 + - -0.32481557 + - -0.005059897 + - -0.23557356 + - -0.039788134 + - -0.1231093 + - 0.7107205 + - -0.3646238 + - 0.681379 + - 0.89263755 + - 0.07968487 + - 0.30849713 + - 0.29674655 + - -0.27216244 + - 0.75820655 + - -0.29331326 + - -0.5996238 + - 0.5672134 + - -0.37904045 + - -0.24415188 + - 0.19066842 + - 1.0094063 + - -0.40063027 + - -0.4825962 + - 0.2774627 + - -0.27590683 + - 0.4612238 + - -0.8983582 + - -0.61800766 + - -0.51829356 + - 0.86593044 + - -0.06357967 + - 0.17033531 + - 0.27793053 + - -0.52404875 + - -0.47101 + - 0.7426831 + - -0.964701 + - 0.34668678 + - -0.033741806 + - -0.29430935 + - 0.7839064 + - 0.95475864 + - -0.074262686 + - -0.115797766 + - 0.34255928 + - 0.6040302 + - -0.15062979 + - -0.56569034 + - -0.4572579 + - -0.28387064 + - -0.5624915 + - 0.40980035 + - -0.15412545 + - -0.26890573 + - 0.06357727 + - -0.7213865 + - -0.1011974 + - 0.11968998 + - 0.19701044 + - -0.33072272 + - -0.15370873 + - -0.30211973 + - -0.25260776 + - -0.49312407 + - -0.37459233 + - 0.011210165 + - 0.18184076 + - -0.03038408 + - -0.6193609 + - 0.21173474 + - 0.22742556 + - 0.16233861 + - -0.088205494 + - 0.49199793 + - 0.29385847 + - 0.89299864 + - -0.59406143 + - 0.7465042 + - 0.006503848 + - 0.34863356 + - -0.62345177 + - -0.45356292 + - -0.63383216 + - -0.14426334 + - 0.070015274 + - 0.03716033 + - 0.98046833 + - 0.28096893 + - 0.07307607 + - 0.32492098 + - -0.37960222 + - -0.4622035 + - 0.39482048 + - 0.06734098 + - 0.31692502 + - -0.049749356 + - 0.1331155 + - 0.28956437 + - 0.0381471 + - -0.38796076 + - 0.04349407 + - 0.008612628 + - -0.24198009 + - -0.090499945 + - 0.18012097 + - 0.23803195 + - -0.5008569 + - 0.15541734 + - 0.52097577 + - 0.33954802 + - -0.5927428 + - -0.025026297 + - 0.15368874 + - 0.077104606 + - 0.16483314 + - 0.9558096 + - 0.34626848 + - -0.45517895 + - -0.39783016 + - 0.2909834 + - 0.5584865 + - 0.059294112 + - 0.13846646 + - 0.42031989 + - 0.010712425 + - 0.021445641 + - -0.45902863 + - -0.6907457 + - -0.002234355 + - -0.030268969 + - 0.2579249 + - 0.7935748 + - -0.25769967 + - -0.1438957 + - 0.26266292 + - 0.28112906 + - -0.34357876 + - 1.0122297 + - -0.2964683 + - 0.3429013 + - -0.5700608 + - -0.09282526 + - 0.4811013 + - 0.35700676 + - -0.16071439 + - 0.11631031 + - -0.16204585 + - -0.47838974 + - -0.49543124 + - 0.026456768 + - 0.5866181 + - -0.20629558 + - 0.5261239 + - -0.26147637 + - -0.82890445 + - -0.05112916 + - -0.13388133 + - -0.45494333 + - 0.45175266 + - 0.28794315 + - 0.9281353 + - 0.5534132 + - 0.24255936 + - 0.2476091 + - 0.47116262 + - -0.2294309 + - -0.059068527 + - -0.12671532 + - -0.23191245 + - 0.35937697 + - 0.36944565 + - 0.3861156 + - 0.04749347 + - 0.09445212 + - 0.7681904 + - 0.5128774 + - -0.46469212 + - -0.41129193 + - -0.14798866 + - -0.026494222 + - 0.1374582 + - 0.1845306 + - -0.41479036 + - -0.4478146 + - -0.027452499 + - -0.39592314 + - -0.3741126 + - -0.19016348 + - 0.39348608 + - 0.007517981 + - 0.37471184 + - -0.12704948 + - -0.46166423 + - -0.14698823 + - -0.06342208 + - 0.037428353 + - 0.07605142 + - -0.3404058 + - 0.46053696 + - 0.56035966 + - -0.55666745 + - -0.35343727 + - 0.06384934 + - -0.14722018 + - -0.225253 + - 0.5212689 + - -0.04948381 + - -0.8933282 + - -0.11180154 + - 0.31227833 + - -0.60436565 + - -0.79203004 + - 0.01730504 + - -0.31900343 + - 0.10426352 + - -0.34517235 + - -0.39474434 + - 0.5102917 + - -0.084621415 + - 0.25734973 + - 0.5436926 + - -0.67924327 + - -0.45912656 + - 0.5325689 + - -0.18137978 + - -0.54084474 + - 0.03943366 + - 0.017366996 + - -1.1773754 + - 0.10030744 + - 0.17339207 + - -0.85046226 + - -0.27805522 + - 0.9682656 + - -0.65961796 + - -0.73597306 + - 0.24702023 + - 0.7053363 + - 0.0749594 + - -0.06025179 + - -0.24279444 + - 0.02613263 + - 0.32312804 + - 0.6360649 + - 0.60153437 + - -0.23882543 + - -0.26499555 + - -1.3346509 + - -0.3764618 + - 0.07609863 + - 0.27458432 + - -0.3821338 + - -0.23214419 + - -0.09689985 + - -0.40563017 + - 0.6572008 + - 0.13450813 + - -0.4460692 + - -0.30316606 + - 0.1958177 + - 0.015910828 + - 0.40730327 + - 0.3669463 + - 0.079425275 + - -0.37171158 + - 0.168909 + - -0.31272405 + - -0.23822992 + - -0.07745864 + - -0.38937923 + - -0.14296639 + - 0.9009801 + - -0.7250084 + - 0.98467064 + - 0.019783655 + - 0.055628438 + - -0.2933434 + - 0.4600183 + - -0.19823411 + - -0.4571724 + - -0.3312746 + - -0.0277608 + - 0.38861823 + - 0.15977266 + - 0.44062296 + - 0.30844745 + - 0.27476984 + - -0.13237253 + - -0.5109521 + - -0.80090564 + - -0.48395592 + - -0.51988745 + - 0.2144506 + - 0.06488733 + - 0.6135665 + - 0.05842493 + - 0.022271514 + - -0.31738955 + - 0.45637342 + - 0.05452357 + - 0.03171132 + - -0.1575031 + - 0.5707459 + - 0.30233765 + - 0.3224614 + - -0.37270218 + - -0.06256089 + - 0.49399772 + - -0.5001368 + - 0.47888866 + - -0.05588177 + - -0.71842545 + - 0.34234884 + - -0.47902894 + - 1.225343 + - -0.6341181 + - 0.09139347 + - 0.04623684 + - 0.3996993 + - 1.1369269 + - 0.83144325 + - -0.13041787 + - -0.19904175 + - 0.36643854 + - 0.402982 + - 0.19888695 + - -0.26939258 + - -0.007989226 + - -0.21660778 + - 0.071891114 + - -0.30885765 + - -0.5136162 + - -0.57677555 + - 0.6942395 + - -0.05473876 + - -0.5325959 + - -0.47834077 + - 0.4090465 + - 0.11244762 + - 0.4355206 + - 0.08062245 + - -0.33711594 + - 0.17177346 + - 0.21119414 + - -0.65168923 + - -1.0551721 + - -0.33327538 + - 0.56057614 + - 0.8861264 + - -0.0225022 + - -0.13925572 + - 0.85244983 + - -0.26732054 + - 0.40896592 + - -0.5453083 + - -0.10937784 + - -0.5180464 + - -1.1474828 + - 0.017952887 + - 0.22886287 + - -0.48226961 + - 0.32168177 + - 0.710783 + - 0.4857719 + - -0.6436394 + - -0.7593292 + - 0.10589559 + - -0.44712177 + - 0.07174561 + - 0.4666989 + - 0.2907941 + - -0.109521724 + - 0.56043 + - 0.11801558 + - 0.4623234 + - -0.7612614 + - 0.08255824 + - 0.56123435 + - -0.22669049 + - 0.46417147 + - -0.12563486 + - -0.17799442 + - -0.3807493 + - 0.08928977 + - 0.37260336 + - 0.010012831 + diff --git a/backends/candle/tests/test_bert.rs b/backends/candle/tests/test_bert.rs new file mode 100644 index 00000000..b195cd48 --- /dev/null +++ b/backends/candle/tests/test_bert.rs @@ -0,0 +1,75 @@ +mod common; + +use crate::common::SnapshotScores; +use anyhow::Result; +use common::{batch, download_artifacts, load_tokenizer, relative_matcher}; +use text_embeddings_backend_candle::CandleBackend; +use text_embeddings_backend_core::{Backend, ModelType, Pool}; + +#[test] +#[serial_test::serial] +fn test_mini() -> Result<()> { + let model_root = download_artifacts("sentence-transformers/all-MiniLM-L6-v2")?; + let tokenizer = load_tokenizer(&model_root)?; + + let backend = CandleBackend::new( + model_root, + "float32".to_string(), + ModelType::Embedding(Pool::Mean), + )?; + + let input_batch = batch(vec![ + tokenizer.encode("What is Deep Learning?", true).unwrap(), + tokenizer.encode("Deep Learning is...", true).unwrap(), + tokenizer.encode("What is Deep Learning?", true).unwrap(), + ]); + + let matcher = relative_matcher(); + + let embeddings_batch = SnapshotScores::from(backend.embed(input_batch)?); + insta::assert_yaml_snapshot!("mini_batch", embeddings_batch, &matcher); + + let input_single = batch(vec![tokenizer + .encode("What is Deep Learning?", true) + .unwrap()]); + + let embeddings_single = SnapshotScores::from(backend.embed(input_single)?); + + insta::assert_yaml_snapshot!("mini_single", embeddings_single, &matcher); + assert_eq!(embeddings_batch[0], embeddings_single[0]); + assert_eq!(embeddings_batch[2], embeddings_single[0]); + + Ok(()) +} + +#[test] +#[serial_test::serial] +fn test_emotions() -> Result<()> { + let model_root = download_artifacts("SamLowe/roberta-base-go_emotions")?; + let tokenizer = load_tokenizer(&model_root)?; + + let backend = CandleBackend::new(model_root, "float32".to_string(), ModelType::Classifier)?; + + let input_batch = batch(vec![ + tokenizer.encode("I like you.", true).unwrap(), + tokenizer + .encode("I am not having a great day.", true) + .unwrap(), + tokenizer.encode("I like you.", true).unwrap(), + ]); + + let matcher = relative_matcher(); + + let predictions_batch = SnapshotScores::from(backend.predict(input_batch)?); + insta::assert_yaml_snapshot!("emotions_batch", predictions_batch, &matcher); + + let input_single = batch(vec![tokenizer.encode("I like you.", true).unwrap()]); + + let predictions_single = SnapshotScores::from(backend.predict(input_single)?); + + insta::assert_yaml_snapshot!("emotions_single", predictions_single, &matcher); + assert_eq!(predictions_batch[0], predictions_single[0]); + assert_eq!(predictions_batch[2], predictions_single[0]); + + Ok(()) +} diff --git a/backends/candle/tests/test_flash_bert.rs b/backends/candle/tests/test_flash_bert.rs new file mode 100644 index 00000000..a053edd6 --- /dev/null +++ b/backends/candle/tests/test_flash_bert.rs @@ -0,0 +1,84 @@ +#![allow(dead_code, unused_imports)] +mod common; + +use crate::common::SnapshotScores; +use anyhow::Result; +use common::{batch, download_artifacts, load_tokenizer, relative_matcher}; +use text_embeddings_backend_candle::CandleBackend; +use text_embeddings_backend_core::{Backend, ModelType, Pool}; + +#[test] +#[serial_test::serial] +#[cfg(all( + feature = "cuda", + any(feature = "flash-attn", feature = "flash-attn-v1") +))] +fn test_flash_mini() -> Result<()> { + let model_root = download_artifacts("sentence-transformers/all-MiniLM-L6-v2")?; + let tokenizer = load_tokenizer(&model_root)?; + + let backend = CandleBackend::new( + model_root, + "float16".to_string(), + ModelType::Embedding(Pool::Mean), + )?; + + let input_batch = batch(vec![ + tokenizer.encode("What is Deep Learning?", true).unwrap(), + tokenizer.encode("Deep Learning is...", true).unwrap(), + tokenizer.encode("What is Deep Learning?", true).unwrap(), + ]); + + let matcher = relative_matcher(); + + let embeddings_batch = SnapshotScores::from(backend.embed(input_batch)?); + insta::assert_yaml_snapshot!("mini_batch", embeddings_batch, &matcher); + + let input_single = batch(vec![tokenizer + .encode("What is Deep Learning?", true) + .unwrap()]); + + let embeddings_single = SnapshotScores::from(backend.embed(input_single)?); + + insta::assert_yaml_snapshot!("mini_single", embeddings_single, &matcher); + assert_eq!(embeddings_batch[0], embeddings_single[0]); + assert_eq!(embeddings_batch[2], embeddings_single[0]); + + Ok(()) +} + +#[test] +#[serial_test::serial] +#[cfg(all( + feature = "cuda", + any(feature = "flash-attn", feature = "flash-attn-v1") +))] +fn test_flash_emotions() -> Result<()> { + let model_root = download_artifacts("SamLowe/roberta-base-go_emotions")?; + let tokenizer = load_tokenizer(&model_root)?; + + let backend = CandleBackend::new(model_root, "float16".to_string(), ModelType::Classifier)?; + + let input_batch = batch(vec![ + tokenizer.encode("I like you.", true).unwrap(), + tokenizer + .encode("I am not having a great day.", true) + .unwrap(), + tokenizer.encode("I like you.", true).unwrap(), + ]); + + let matcher = relative_matcher(); + + let predictions_batch = SnapshotScores::from(backend.predict(input_batch)?); + insta::assert_yaml_snapshot!("emotions_batch", predictions_batch, &matcher); + + let input_single = batch(vec![tokenizer.encode("I like you.", true).unwrap()]); + + let predictions_single = SnapshotScores::from(backend.predict(input_single)?); + + insta::assert_yaml_snapshot!("emotions_single", predictions_single, &matcher); + assert_eq!(predictions_batch[0], predictions_single[0]); + assert_eq!(predictions_batch[2], predictions_single[0]); + + Ok(()) +} diff --git a/backends/candle/tests/test_jina.rs b/backends/candle/tests/test_jina.rs new file mode 100644 index 00000000..bc1b58bd --- /dev/null +++ b/backends/candle/tests/test_jina.rs @@ -0,0 +1,42 @@ +mod common; + +use crate::common::SnapshotScores; +use anyhow::Result; +use common::{batch, download_artifacts, load_tokenizer, relative_matcher}; +use text_embeddings_backend_candle::CandleBackend; +use text_embeddings_backend_core::{Backend, ModelType, Pool}; + +#[test] +fn test_jina_small() -> Result<()> { + let model_root = download_artifacts("jinaai/jina-embeddings-v2-small-en")?; + let tokenizer = load_tokenizer(&model_root)?; + + let backend = CandleBackend::new( + model_root, + "float32".to_string(), + ModelType::Embedding(Pool::Mean), + )?; + + let input_batch = batch(vec![ + tokenizer.encode("What is Deep Learning?", true).unwrap(), + tokenizer.encode("Deep Learning is...", true).unwrap(), + tokenizer.encode("What is Deep Learning?", true).unwrap(), + ]); + + let matcher = relative_matcher(); + + let embeddings_batch = SnapshotScores::from(backend.embed(input_batch)?); + insta::assert_yaml_snapshot!("jina_batch", embeddings_batch, &matcher); + + let input_single = batch(vec![tokenizer + .encode("What is Deep Learning?", true) + .unwrap()]); + + let embeddings_single = SnapshotScores::from(backend.embed(input_single)?); + + insta::assert_yaml_snapshot!("jina_single", embeddings_single, &matcher); + assert_eq!(embeddings_batch[0], embeddings_single[0]); + assert_eq!(embeddings_batch[2], embeddings_single[0]); + + Ok(()) +} diff --git a/backends/src/lib.rs b/backends/src/lib.rs index 7eca2be5..0d32cb95 100644 --- a/backends/src/lib.rs +++ b/backends/src/lib.rs @@ -143,7 +143,7 @@ fn init_backend( #[cfg(feature = "python")] { return Ok(Box::new( - thread::spawn(move || { + std::thread::spawn(move || { PythonBackend::new( model_path.to_str().unwrap().to_string(), dtype.to_string(), diff --git a/router/Cargo.toml b/router/Cargo.toml index 32d130b9..d87bd724 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -54,6 +54,11 @@ tonic-health = { version = "0.10.2", optional = true } tonic-reflection = { version = "0.10.2", optional = true } tokio-stream = { version = "0.1.14", optional = true } +[dev-dependencies] +insta = { git = "https://github.com/OlivierDehaene/insta", rev = "f4f98c0410b91fb5a28b10df98e4422955be9c2c", features = ["yaml"] } +is_close = "0.1.3" +reqwest = { version = "0.11.22", features = ["json"] } + [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } tonic-build = { version = "0.10.2", optional = true } diff --git a/router/src/lib.rs b/router/src/lib.rs index d7164707..de3e902e 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,42 +1,324 @@ -use ::http::HeaderMap; /// Text Embedding Inference Webserver -use anyhow::Result; +mod logging; +mod prometheus; + +#[cfg(feature = "http")] +mod http; + +#[cfg(feature = "grpc")] +mod grpc; +mod shutdown; + +use ::http::HeaderMap; +use anyhow::{anyhow, Context, Result}; +use hf_hub::api::tokio::ApiBuilder; +use hf_hub::{Repo, RepoType}; +use serde::Deserialize; use serde::Serialize; use std::collections::HashMap; -use std::net::SocketAddr; +use std::fs; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::path::Path; use std::time::{Duration, Instant}; +use text_embeddings_backend::DType; +use text_embeddings_core::download::{download_artifacts, download_pool_config}; use text_embeddings_core::infer::Infer; +use text_embeddings_core::queue::Queue; +use text_embeddings_core::tokenization::Tokenization; use text_embeddings_core::TextEmbeddingsError; +use tokenizers::decoders::metaspace::PrependScheme; +use tokenizers::{PreTokenizerWrapper, Tokenizer}; use tracing::Span; -mod prometheus; +pub use logging::init_logging; -#[cfg(feature = "http")] -mod http; +/// Create entrypoint +#[allow(clippy::too_many_arguments)] +pub async fn run( + model_id: String, + revision: Option, + tokenization_workers: Option, + dtype: Option, + pooling: Option, + max_concurrent_requests: usize, + max_batch_tokens: usize, + max_batch_requests: Option, + max_client_batch_size: usize, + hf_api_token: Option, + hostname: Option, + port: u16, + uds_path: Option, + huggingface_hub_cache: Option, + otlp_endpoint: Option, +) -> Result<()> { + let model_id_path = Path::new(&model_id); + let model_root = if model_id_path.exists() && model_id_path.is_dir() { + // Using a local model + model_id_path.to_path_buf() + } else { + let mut builder = ApiBuilder::new() + .with_progress(false) + .with_token(hf_api_token); -#[cfg(feature = "grpc")] -mod grpc; -mod shutdown; + if let Some(cache_dir) = huggingface_hub_cache { + builder = builder.with_cache_dir(cache_dir.into()); + } -/// Crate entrypoint -pub async fn run(infer: Infer, info: Info, addr: SocketAddr) -> Result<()> { - let prom_builder = prometheus::prometheus_builer(info.max_input_length)?; + let api = builder.build().unwrap(); + let api_repo = api.repo(Repo::with_revision( + model_id.clone(), + RepoType::Model, + revision.clone().unwrap_or("main".to_string()), + )); - if cfg!(feature = "http") { - #[cfg(feature = "http")] - { - return http::server::run(infer, info, addr, prom_builder).await; + // Optionally download the pooling config. + if pooling.is_none() { + // If a pooling config exist, download it + let _ = download_pool_config(&api_repo).await; + } + + // Download model from the Hub + download_artifacts(&api_repo) + .await + .context("Could not download model artifacts")? + }; + + // Load config + let config_path = model_root.join("config.json"); + let config = fs::read_to_string(config_path).context("`config.json` not found")?; + let config: ModelConfig = + serde_json::from_str(&config).context("Failed to parse `config.json`")?; + + // Set model type from config + let backend_model_type = { + // Check if the model is a classifier + let mut classifier = false; + for arch in &config.architectures { + if arch.ends_with("Classification") { + classifier = true; + break; + } + } + + if classifier { + if pooling.is_some() { + tracing::warn!( + "`--pooling` arg is set but model is a classifier. Ignoring `--pooling` arg." + ); + } + text_embeddings_backend::ModelType::Classifier + } else { + // Set pooling + let pool = match pooling { + Some(pool) => pool, + None => { + // Load pooling config + let config_path = model_root.join("1_Pooling/config.json"); + let config = fs::read_to_string(config_path).context("The `--pooling` arg is not set and we could not find a pooling configuration (`1_Pooling/config.json`) for this model.")?; + let config: PoolConfig = serde_json::from_str(&config) + .context("Failed to parse `1_Pooling/config.json`")?; + if config.pooling_mode_cls_token { + text_embeddings_backend::Pool::Cls + } else if config.pooling_mode_mean_tokens { + text_embeddings_backend::Pool::Mean + } else { + return Err(anyhow!("Pooling config {config:?} is not supported")); + } + } + }; + text_embeddings_backend::ModelType::Embedding(pool) + } + }; + + // Info model type + let model_type = match &backend_model_type { + text_embeddings_backend::ModelType::Classifier => { + let id2label = config + .id2label + .context("`config.json` does not contain `id2label`")?; + let n_classes = id2label.len(); + let classifier_model = ClassifierModel { + id2label, + label2id: config + .label2id + .context("`config.json` does not contain `label2id`")?, + }; + if n_classes > 1 { + ModelType::Classifier(classifier_model) + } else { + ModelType::Reranker(classifier_model) + } + } + text_embeddings_backend::ModelType::Embedding(pool) => { + ModelType::Embedding(EmbeddingModel { + pooling: pool.to_string(), + }) + } + }; + + // Load tokenizer + let tokenizer_path = model_root.join("tokenizer.json"); + let mut tokenizer = Tokenizer::from_file(tokenizer_path).expect( + "tokenizer.json not found. text-embeddings-inference only supports fast tokenizers", + ); + // See https://github.com/huggingface/tokenizers/pull/1357 + if let Some(pre_tokenizer) = tokenizer.get_pre_tokenizer() { + if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer { + // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer` + let mut m = m.clone(); + m.set_prepend_scheme(PrependScheme::First); + tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Metaspace(m)); + } else if let PreTokenizerWrapper::Sequence(s) = pre_tokenizer { + // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer` + let mut s = s.clone(); + for pre_tokenizer in s.get_pre_tokenizers_mut() { + if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer { + m.set_prepend_scheme(PrependScheme::First); + } + } + tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Sequence(s)); } } - if cfg!(feature = "grpc") { - #[cfg(feature = "grpc")] + tokenizer.with_padding(None); + + // Position IDs offset. Used for Roberta and camembert. + let position_offset = if &config.model_type == "xlm-roberta" + || &config.model_type == "camembert" + || &config.model_type == "roberta" + { + config.pad_token_id + 1 + } else { + 0 + }; + let max_input_length = config.max_position_embeddings - position_offset; + + let tokenization_workers = tokenization_workers.unwrap_or_else(num_cpus::get_physical); + + // Tokenization logic + let tokenization = Tokenization::new( + tokenization_workers, + tokenizer, + max_input_length, + position_offset, + ); + + // Get dtype + let dtype = dtype.unwrap_or({ + #[cfg(any(feature = "accelerate", feature = "mkl", feature = "mkl-dynamic"))] { - return grpc::server::run(infer, info, addr, prom_builder).await; + DType::Float32 } + #[cfg(not(any(feature = "accelerate", feature = "mkl", feature = "mkl-dynamic")))] + { + DType::Float16 + } + }); + + // Create backend + tracing::info!("Starting model backend"); + let backend = text_embeddings_backend::Backend::new( + model_root, + dtype.clone(), + backend_model_type, + uds_path.unwrap_or("/tmp/text-embeddings-inference-server".to_string()), + otlp_endpoint.clone(), + ) + .context("Could not create backend")?; + backend + .health() + .await + .context("Model backend is not healthy")?; + + let max_batch_requests = backend + .max_batch_size + .map(|s| { + tracing::warn!("Backend does not support a batch size > {s}"); + tracing::warn!("forcing `max_batch_requests={s}`"); + s + }) + .or(max_batch_requests); + + // Queue logic + let queue = Queue::new( + backend.padded_model, + max_batch_tokens, + max_batch_requests, + max_concurrent_requests, + ); + + // Create infer task + let infer = Infer::new(tokenization, queue, max_concurrent_requests, backend); + + // Endpoint info + let info = Info { + model_id, + model_sha: revision, + model_dtype: dtype.to_string(), + model_type, + max_concurrent_requests, + max_input_length, + max_batch_tokens, + tokenization_workers, + max_batch_requests, + max_client_batch_size, + version: env!("CARGO_PKG_VERSION"), + sha: option_env!("VERGEN_GIT_SHA"), + docker_label: option_env!("DOCKER_LABEL"), + }; + + let addr = match hostname.unwrap_or("0.0.0.0".to_string()).parse() { + Ok(ip) => SocketAddr::new(ip, port), + Err(_) => { + tracing::warn!("Invalid hostname, defaulting to 0.0.0.0"); + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port) + } + }; + + let prom_builder = prometheus::prometheus_builer(info.max_input_length)?; + + #[cfg(all(feature = "grpc", feature = "http"))] + compile_error!("Features `http` and `grpc` cannot be enabled at the same time."); + + #[cfg(not(any(feature = "http", feature = "grpc")))] + compile_error!("Either feature `http` or `grpc` must be enabled."); + + #[cfg(feature = "http")] + { + let server = + tokio::spawn(async move { http::server::run(infer, info, addr, prom_builder).await }); + tracing::info!("Ready"); + server.await??; } - anyhow::bail!("You must use one of `http` or `grpc`"); + #[cfg(feature = "grpc")] + { + let server = + tokio::spawn(async move { grpc::server::run(infer, info, addr, prom_builder).await }); + tracing::info!("Ready"); + server.await??; + } + + Ok(()) +} + +#[derive(Debug, Deserialize)] +pub struct ModelConfig { + pub architectures: Vec, + pub model_type: String, + #[serde(alias = "n_positions")] + pub max_position_embeddings: usize, + pub pad_token_id: usize, + pub id2label: Option>, + pub label2id: Option>, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct PoolConfig { + pooling_mode_cls_token: bool, + pooling_mode_mean_tokens: bool, + pooling_mode_max_tokens: bool, + pooling_mode_mean_sqrt_len_tokens: bool, } #[derive(Clone, Debug, Serialize)] diff --git a/router/src/logging.rs b/router/src/logging.rs new file mode 100644 index 00000000..f8a6f0aa --- /dev/null +++ b/router/src/logging.rs @@ -0,0 +1,65 @@ +use opentelemetry::sdk::propagation::TraceContextPropagator; +use opentelemetry::sdk::trace::Sampler; +use opentelemetry::sdk::{trace, Resource}; +use opentelemetry::{global, KeyValue}; +use opentelemetry_otlp::WithExportConfig; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{EnvFilter, Layer}; + +/// Init logging using env variables LOG_LEVEL and LOG_FORMAT: +/// - otlp_endpoint is an optional URL to an Open Telemetry collector +/// - LOG_LEVEL may be TRACE, DEBUG, INFO, WARN or ERROR (default to INFO) +pub fn init_logging(otlp_endpoint: Option<&String>, json_output: bool) -> bool { + let mut layers = Vec::new(); + + // STDOUT/STDERR layer + let fmt_layer = tracing_subscriber::fmt::layer() + .with_file(true) + .with_line_number(true); + + let fmt_layer = match json_output { + true => fmt_layer.json().flatten_event(true).boxed(), + false => fmt_layer.boxed(), + }; + layers.push(fmt_layer); + + // OpenTelemetry tracing layer + let mut global_tracer = false; + if let Some(otlp_endpoint) = otlp_endpoint { + global::set_text_map_propagator(TraceContextPropagator::new()); + + let tracer = opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint(otlp_endpoint), + ) + .with_trace_config( + trace::config() + .with_resource(Resource::new(vec![KeyValue::new( + "service.name", + "text-embeddings-inference.router", + )])) + .with_sampler(Sampler::AlwaysOn), + ) + .install_batch(opentelemetry::runtime::Tokio); + + if let Ok(tracer) = tracer { + layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed()); + init_tracing_opentelemetry::init_propagator().unwrap(); + global_tracer = true; + }; + } + + // Filter events with LOG_LEVEL + let env_filter = + EnvFilter::try_from_env("LOG_LEVEL").unwrap_or_else(|_| EnvFilter::new("info")); + + tracing_subscriber::registry() + .with(env_filter) + .with(layers) + .init(); + global_tracer +} diff --git a/router/src/main.rs b/router/src/main.rs index 22d59a17..7323cb34 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,29 +1,7 @@ -use anyhow::{anyhow, Context, Result}; +use anyhow::Result; use clap::Parser; -use hf_hub::api::tokio::ApiBuilder; -use hf_hub::{Repo, RepoType}; -use opentelemetry::sdk::propagation::TraceContextPropagator; -use opentelemetry::sdk::trace::Sampler; -use opentelemetry::sdk::{trace, Resource}; -use opentelemetry::{global, KeyValue}; -use opentelemetry_otlp::WithExportConfig; -use serde::Deserialize; -use std::collections::HashMap; -use std::fs; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::path::Path; +use opentelemetry::global; use text_embeddings_backend::DType; -use text_embeddings_core::download::{download_artifacts, download_pool_config}; -use text_embeddings_core::infer::Infer; -use text_embeddings_core::queue::Queue; -use text_embeddings_core::tokenization::Tokenization; -use text_embeddings_router::{ClassifierModel, EmbeddingModel, Info, ModelType}; -use tokenizers::decoders::metaspace::PrependScheme; -use tokenizers::pre_tokenizers::sequence::Sequence; -use tokenizers::{PreTokenizerWrapper, Tokenizer}; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; -use tracing_subscriber::{EnvFilter, Layer}; use veil::Redact; /// App Configuration @@ -122,347 +100,39 @@ struct Args { otlp_endpoint: Option, } -#[derive(Debug, Deserialize)] -pub struct ModelConfig { - pub architectures: Vec, - pub model_type: String, - #[serde(alias = "n_positions")] - pub max_position_embeddings: usize, - pub pad_token_id: usize, - pub id2label: Option>, - pub label2id: Option>, -} - -#[derive(Debug, Clone, PartialEq, Deserialize)] -pub struct PoolConfig { - pooling_mode_cls_token: bool, - pooling_mode_mean_tokens: bool, - pooling_mode_max_tokens: bool, - pooling_mode_mean_sqrt_len_tokens: bool, -} - #[tokio::main] async fn main() -> Result<()> { // Pattern match configuration let args: Args = Args::parse(); - // Initialize loggin and telemetry - init_logging(args.otlp_endpoint.clone(), args.json_output); + // Initialize logging and telemetry + let global_tracer = + text_embeddings_router::init_logging(args.otlp_endpoint.as_ref(), args.json_output); tracing::info!("{args:?}"); - let model_id_path = Path::new(&args.model_id); - let model_root = if model_id_path.exists() && model_id_path.is_dir() { - // Using a local model - model_id_path.to_path_buf() - } else { - let mut builder = ApiBuilder::new() - .with_progress(false) - .with_token(args.hf_api_token); - - if let Some(cache_dir) = args.huggingface_hub_cache { - builder = builder.with_cache_dir(cache_dir.into()); - } - - let api = builder.build().unwrap(); - let api_repo = api.repo(Repo::with_revision( - args.model_id.clone(), - RepoType::Model, - args.revision.clone().unwrap_or("main".to_string()), - )); - - // Optionally download the pooling config. - if args.pooling.is_none() { - // If a pooling config exist, download it - let _ = download_pool_config(&api_repo).await; - } - - // Download model from the Hub - download_artifacts(&api_repo) - .await - .context("Could not download model artifacts")? - }; - - // Load config - let config_path = model_root.join("config.json"); - let config = fs::read_to_string(config_path).context("`config.json` not found")?; - let config: ModelConfig = - serde_json::from_str(&config).context("Failed to parse `config.json`")?; - - // Set model type from config - let backend_model_type = { - // Check if the model is a classifier - let mut classifier = false; - for arch in &config.architectures { - if arch.ends_with("Classification") { - classifier = true; - break; - } - } - - if classifier { - if args.pooling.is_some() { - tracing::warn!( - "`--pooling` arg is set but model is a classifier. Ignoring `--pooling` arg." - ); - } - text_embeddings_backend::ModelType::Classifier - } else { - // Set pooling - let pool = match args.pooling { - Some(pool) => pool, - None => { - // Load pooling config - let config_path = model_root.join("1_Pooling/config.json"); - let config = fs::read_to_string(config_path).context("The `--pooling` arg is not set and we could not find a pooling configuration (`1_Pooling/config.json`) for this model.")?; - let config: PoolConfig = serde_json::from_str(&config) - .context("Failed to parse `1_Pooling/config.json`")?; - if config.pooling_mode_cls_token { - text_embeddings_backend::Pool::Cls - } else if config.pooling_mode_mean_tokens { - text_embeddings_backend::Pool::Mean - } else { - return Err(anyhow!("Pooling config {config:?} is not supported")); - } - } - }; - text_embeddings_backend::ModelType::Embedding(pool) - } - }; - - // Info model type - let model_type = match &backend_model_type { - text_embeddings_backend::ModelType::Classifier => { - let id2label = config - .id2label - .context("`config.json` does not contain `id2label`")?; - let n_classes = id2label.len(); - let classifier_model = ClassifierModel { - id2label, - label2id: config - .label2id - .context("`config.json` does not contain `label2id`")?, - }; - if n_classes > 1 { - ModelType::Classifier(classifier_model) - } else { - ModelType::Reranker(classifier_model) - } - } - text_embeddings_backend::ModelType::Embedding(pool) => { - ModelType::Embedding(EmbeddingModel { - pooling: pool.to_string(), - }) - } - }; - - // Load tokenizer - let tokenizer_path = model_root.join("tokenizer.json"); - let mut tokenizer = Tokenizer::from_file(tokenizer_path).expect( - "tokenizer.json not found. text-embeddings-inference only supports fast tokenizers", - ); - // See https://github.com/huggingface/tokenizers/pull/1357 - if let Some(pre_tokenizer) = tokenizer.get_pre_tokenizer() { - if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer { - // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer` - let mut m = m.clone(); - m.set_prepend_scheme(PrependScheme::First); - tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Metaspace(m)); - } else if let PreTokenizerWrapper::Sequence(s) = pre_tokenizer { - let pre_tokenizers = s.get_pre_tokenizers(); - // Check if we have a Metaspace pre tokenizer in the sequence - let has_metaspace = pre_tokenizers - .iter() - .find(|t| matches!(t, PreTokenizerWrapper::Metaspace(_))) - .is_some(); - - if has_metaspace { - let mut new_pre_tokenizers = Vec::with_capacity(s.get_pre_tokenizers().len()); - - for pre_tokenizer in pre_tokenizers { - if let PreTokenizerWrapper::WhitespaceSplit(_) = pre_tokenizer { - // Remove WhitespaceSplit - // This will be done by the Metaspace pre tokenizer - continue; - } - - let mut pre_tokenizer = pre_tokenizer.clone(); - - if let PreTokenizerWrapper::Metaspace(ref mut m) = pre_tokenizer { - m.set_prepend_scheme(PrependScheme::First); - } - new_pre_tokenizers.push(pre_tokenizer); - } - tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Sequence(Sequence::new( - new_pre_tokenizers, - ))); - } - } - } - - tokenizer.with_padding(None); - - // Position IDs offset. Used for Roberta and camembert. - let position_offset = if &config.model_type == "xlm-roberta" - || &config.model_type == "camembert" - || &config.model_type == "roberta" - { - config.pad_token_id + 1 - } else { - 0 - }; - let max_input_length = config.max_position_embeddings - position_offset; - - let tokenization_workers = args - .tokenization_workers - .unwrap_or_else(num_cpus::get_physical); - - // Tokenization logic - let tokenization = Tokenization::new( - tokenization_workers, - tokenizer, - max_input_length, - position_offset, - ); - - // Get dtype - let dtype = args.dtype.unwrap_or({ - #[cfg(any(feature = "accelerate", feature = "mkl", feature = "mkl-dynamic"))] - { - DType::Float32 - } - #[cfg(not(any(feature = "accelerate", feature = "mkl", feature = "mkl-dynamic")))] - { - DType::Float16 - } - }); - - // Create backend - tracing::info!("Starting model backend"); - let backend = text_embeddings_backend::Backend::new( - model_root, - dtype.clone(), - backend_model_type, - args.uds_path, - args.otlp_endpoint.clone(), - ) - .context("Could not create backend")?; - backend - .health() - .await - .context("Model backend is not healthy")?; - - let max_batch_requests = backend - .max_batch_size - .map(|s| { - tracing::warn!("Backend does not support a batch size > {s}"); - tracing::warn!("forcing `max_batch_requests={s}`"); - s - }) - .or(args.max_batch_requests); - - // Queue logic - let queue = Queue::new( - backend.padded_model, - args.max_batch_tokens, - max_batch_requests, + text_embeddings_router::run( + args.model_id, + args.revision, + args.tokenization_workers, + args.dtype, + args.pooling, args.max_concurrent_requests, - ); - - // Create infer task - let infer = Infer::new(tokenization, queue, args.max_concurrent_requests, backend); - - // Endpoint info - let info = Info { - model_id: args.model_id, - model_sha: args.revision, - model_dtype: dtype.to_string(), - model_type, - max_concurrent_requests: args.max_concurrent_requests, - max_input_length, - max_batch_tokens: args.max_batch_tokens, - tokenization_workers, - max_batch_requests, - max_client_batch_size: args.max_client_batch_size, - version: env!("CARGO_PKG_VERSION"), - sha: option_env!("VERGEN_GIT_SHA"), - docker_label: option_env!("DOCKER_LABEL"), - }; - - let addr = match args.hostname.parse() { - Ok(ip) => SocketAddr::new(ip, args.port), - Err(_) => { - tracing::warn!("Invalid hostname, defaulting to 0.0.0.0"); - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), args.port) - } - }; - - tracing::info!("Ready"); - - // Run axum server - text_embeddings_router::run(infer, info, addr) - .await - .unwrap(); + args.max_batch_tokens, + args.max_batch_requests, + args.max_client_batch_size, + args.hf_api_token, + Some(args.hostname), + args.port, + Some(args.uds_path), + args.huggingface_hub_cache, + args.otlp_endpoint, + ) + .await?; - if args.otlp_endpoint.is_some() { + if global_tracer { // Shutdown tracer global::shutdown_tracer_provider(); } - Ok(()) } - -/// Init logging using env variables LOG_LEVEL and LOG_FORMAT: -/// - otlp_endpoint is an optional URL to an Open Telemetry collector -/// - LOG_LEVEL may be TRACE, DEBUG, INFO, WARN or ERROR (default to INFO) -/// - LOG_FORMAT may be TEXT or JSON (default to TEXT) -fn init_logging(otlp_endpoint: Option, json_output: bool) { - let mut layers = Vec::new(); - - // STDOUT/STDERR layer - let fmt_layer = tracing_subscriber::fmt::layer() - .with_file(true) - .with_line_number(true); - - let fmt_layer = match json_output { - true => fmt_layer.json().flatten_event(true).boxed(), - false => fmt_layer.boxed(), - }; - layers.push(fmt_layer); - - // OpenTelemetry tracing layer - if let Some(otlp_endpoint) = otlp_endpoint { - global::set_text_map_propagator(TraceContextPropagator::new()); - - let tracer = opentelemetry_otlp::new_pipeline() - .tracing() - .with_exporter( - opentelemetry_otlp::new_exporter() - .tonic() - .with_endpoint(otlp_endpoint), - ) - .with_trace_config( - trace::config() - .with_resource(Resource::new(vec![KeyValue::new( - "service.name", - "text-embeddings-inference.router", - )])) - .with_sampler(Sampler::AlwaysOn), - ) - .install_batch(opentelemetry::runtime::Tokio); - - if let Ok(tracer) = tracer { - layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed()); - init_tracing_opentelemetry::init_propagator().unwrap(); - }; - } - - // Filter events with LOG_LEVEL - let env_filter = - EnvFilter::try_from_env("LOG_LEVEL").unwrap_or_else(|_| EnvFilter::new("info")); - - tracing_subscriber::registry() - .with(env_filter) - .with(layers) - .init(); -} diff --git a/router/src/prometheus.rs b/router/src/prometheus.rs index 3c16684e..63fcbe74 100644 --- a/router/src/prometheus.rs +++ b/router/src/prometheus.rs @@ -1,6 +1,6 @@ use metrics_exporter_prometheus::{BuildError, Matcher, PrometheusBuilder}; -pub fn prometheus_builer(max_input_length: usize) -> Result { +pub(crate) fn prometheus_builer(max_input_length: usize) -> Result { // Duration buckets let duration_matcher = Matcher::Suffix(String::from("duration")); let n_duration_buckets = 35; diff --git a/router/tests/common.rs b/router/tests/common.rs new file mode 100644 index 00000000..b4b01166 --- /dev/null +++ b/router/tests/common.rs @@ -0,0 +1,72 @@ +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::time::Duration; +use text_embeddings_backend::DType; +use text_embeddings_router::run; +use tokio::time::Instant; + +#[derive(Serialize, Deserialize, Debug)] +pub struct Score(f32); + +impl Score { + fn is_close(&self, other: &Self, abs_tol: f32) -> bool { + is_close::default() + .abs_tol(abs_tol) + .is_close(self.0, other.0) + } +} + +impl PartialEq for Score { + fn eq(&self, other: &Self) -> bool { + // Default tolerance for equality + self.is_close(other, 4e-3) + } +} + +async fn check_health(port: u16, timeout: Duration) -> Result<()> { + let addr = format!("http://0.0.0.0:{port}/health"); + let client = reqwest::ClientBuilder::new() + .timeout(timeout) + .build() + .unwrap(); + + let start = Instant::now(); + loop { + if client.get(&addr).send().await.is_ok() { + return Ok(()); + } + if start.elapsed() < timeout { + tokio::time::sleep(Duration::from_secs(1)).await; + } else { + anyhow::bail!("Backend is not healthy"); + } + } +} + +pub async fn start_server(model_id: String, revision: Option, dtype: DType) -> Result<()> { + let server_task = tokio::spawn({ + run( + model_id, + revision, + Some(1), + Some(dtype), + None, + 4, + 1024, + None, + 32, + None, + None, + 8090, + None, + None, + None, + ) + }); + + tokio::select! { + err = server_task => err?, + _ = check_health(8090, Duration::from_secs(60)) => Ok(()) + }?; + Ok(()) +} diff --git a/router/tests/snapshots/test_http_embed__embeddings_batch.snap b/router/tests/snapshots/test_http_embed__embeddings_batch.snap new file mode 100644 index 00000000..41361c46 --- /dev/null +++ b/router/tests/snapshots/test_http_embed__embeddings_batch.snap @@ -0,0 +1,1925 @@ +--- +source: router/tests/test_http_embed.rs +expression: embeddings_batch +--- +- - 0.011566745 + - 0.025191609 + - -0.036694292 + - 0.05925345 + - -0.007202666 + - -0.041202188 + - 0.07718597 + - 0.037412453 + - 0.012524345 + - -0.006153889 + - 0.017013893 + - -0.07700467 + - -0.00042374025 + - 0.027820537 + - -0.015982956 + - -0.06823823 + - 0.0088968985 + - -0.020330165 + - -0.080409184 + - -0.013094234 + - -0.04111577 + - -0.02575625 + - -0.026604561 + - 0.033058524 + - -0.022127466 + - 0.021035336 + - -0.057914518 + - 0.033003185 + - 0.02975051 + - -0.062229656 + - 0.038802397 + - 0.03196761 + - 0.0153276855 + - 0.045363918 + - 0.05309402 + - 0.013351721 + - 0.041196864 + - 0.028114794 + - 0.019435482 + - -0.0032642381 + - -0.0035951673 + - -0.14286515 + - 0.03804074 + - -0.010979397 + - 0.02610546 + - 0.041429143 + - -0.016078437 + - 0.05356371 + - -0.05677031 + - 0.012202429 + - -0.03505195 + - -0.039740372 + - -0.046098847 + - -0.03911731 + - -0.018020066 + - 0.021650145 + - -0.0064859986 + - -0.026541203 + - 0.04864548 + - 0.04339141 + - 0.04626493 + - -0.03446706 + - -0.024146007 + - 0.055675633 + - 0.024611346 + - 0.019454936 + - 0.011295259 + - -0.025712365 + - -0.03224502 + - -0.032699518 + - -0.014605892 + - 0.0146714635 + - 0.010346884 + - 0.07415931 + - 0.080031686 + - -0.0042066616 + - 0.0026997656 + - -0.076720476 + - 0.05881217 + - -0.004803097 + - -0.087379314 + - -0.046413604 + - -0.039611273 + - 0.052492607 + - 0.024743188 + - 0.080671504 + - 0.1198378 + - 0.03179342 + - -0.11392431 + - 0.010694122 + - 0.01976001 + - 0.03866714 + - -0.034617126 + - -0.0061281016 + - -0.042578902 + - 0.022671003 + - 0.008379255 + - -0.019488879 + - -0.023300469 + - 0.24531509 + - 0.049447276 + - 0.029196257 + - -0.0057606567 + - -0.020439675 + - -0.07763772 + - -0.043004043 + - 0.0003997896 + - -0.066854045 + - 0.07072114 + - 0.0064844415 + - -0.058493868 + - -0.005078577 + - 0.030898178 + - 0.027523836 + - 0.027156036 + - -0.06816291 + - -0.06572996 + - 0.059115347 + - -0.031852506 + - 0.040637866 + - 0.057000894 + - 0.0069915373 + - 0.0062051546 + - -0.013209068 + - 0.031209558 + - -0.04034961 + - 0.070826106 + - -0.0000000000000000000000000000000048484194 + - 0.021934593 + - -0.10270966 + - 0.05622236 + - 0.09768862 + - -0.052761998 + - 0.019099213 + - -0.010433081 + - 0.07028384 + - -0.009163081 + - 0.059657086 + - 0.0093431 + - -0.015467531 + - -0.023266213 + - 0.023969134 + - 0.10212125 + - 0.09321899 + - -0.033179756 + - 0.011723363 + - -0.065606475 + - 0.031413328 + - -0.0237615 + - -0.0486851 + - 0.012292702 + - -0.0400642 + - -0.076479904 + - -0.035577532 + - -0.0013617877 + - -0.016482001 + - 0.016856715 + - -0.0004216753 + - -0.05272671 + - 0.029926106 + - -0.07283276 + - 0.06911143 + - -0.017620781 + - -0.005552996 + - 0.012905943 + - -0.022742389 + - 0.026754798 + - -0.025787508 + - -0.04020589 + - -0.013465992 + - 0.00076997397 + - 0.028548837 + - 0.031916283 + - -0.031540327 + - -0.029507546 + - -0.020180529 + - 0.048006866 + - -0.0013082745 + - -0.013749675 + - 0.020036789 + - -0.068734996 + - -0.021904256 + - -0.031283367 + - 0.049285103 + - 0.012069677 + - -0.058844592 + - -0.026464054 + - 0.05979334 + - 0.067720294 + - 0.034020975 + - -0.05295764 + - 0.059824925 + - -0.02547595 + - -0.020665556 + - -0.053789217 + - -0.09736991 + - 0.04800511 + - 0.05235638 + - -0.023362491 + - -0.06907576 + - 0.016694613 + - 0.02856358 + - -0.029173644 + - -0.03551531 + - -0.0126147205 + - 0.07328874 + - -0.019482115 + - -0.063127376 + - 0.096063904 + - -0.07734705 + - 0.015912902 + - -0.044809945 + - 0.016362015 + - -0.0007529947 + - -0.008698364 + - -0.098845914 + - 0.005773373 + - -0.071921706 + - -0.032704372 + - 0.019982614 + - 0.003827613 + - -0.025586382 + - 0.08240085 + - 0.0000000000000000000000000000000040850666 + - -0.029535603 + - 0.025522647 + - -0.051075805 + - 0.1553659 + - 0.0523394 + - -0.034587402 + - 0.13312408 + - -0.019211281 + - -0.0597844 + - 0.12298254 + - 0.010122296 + - -0.049751673 + - 0.058429115 + - 0.012729909 + - -0.016639234 + - 0.012746897 + - 0.04571337 + - -0.06986801 + - -0.04855961 + - -0.0049983296 + - -0.09038651 + - 0.069924004 + - 0.009439225 + - -0.0067679146 + - -0.106080666 + - 0.03100246 + - 0.04949134 + - -0.044834208 + - -0.007470489 + - -0.033606272 + - 0.07610607 + - 0.0071234847 + - -0.042222 + - 0.0708183 + - 0.047470734 + - 0.020847816 + - 0.15337874 + - -0.008404033 + - -0.025955822 + - 0.06075936 + - 0.06673509 + - 0.064682245 + - 0.049773723 + - 0.08875065 + - -0.0329716 + - 0.0703123 + - 0.017220959 + - -0.030130997 + - 0.03847389 + - 0.04841252 + - -0.060448505 + - 0.03055366 + - 0.015585768 + - -0.030456763 + - -0.009485868 + - -0.041138977 + - -0.06799712 + - 0.010231143 + - -0.025668688 + - 0.021700189 + - -0.06986792 + - 0.092408456 + - -0.03572349 + - 0.07020521 + - -0.06350947 + - -0.0328256 + - -0.046145674 + - 0.054110475 + - 0.051667217 + - 0.042987943 + - 0.013467606 + - 0.016568488 + - -0.04419887 + - -0.019691551 + - 0.03615324 + - -0.019701054 + - -0.1157151 + - 0.0059685465 + - 0.0045423866 + - -0.045008406 + - -0.06848648 + - -0.0853551 + - -0.07097772 + - 0.08044467 + - -0.057986956 + - 0.057813518 + - 0.050183795 + - 0.05942627 + - -0.036489185 + - 0.009364899 + - 0.052608218 + - 0.028058298 + - -0.033277407 + - -0.050756633 + - -0.01291192 + - -0.0000000142961065 + - -0.04050862 + - -0.085779026 + - 0.04517046 + - 0.021616641 + - -0.02237283 + - 0.012219842 + - -0.0324641 + - -0.016899578 + - -0.027103093 + - 0.005931705 + - 0.04030224 + - 0.026972173 + - -0.035654355 + - 0.07405438 + - 0.032423433 + - -0.090567395 + - -0.0316623 + - 0.040873803 + - -0.009927164 + - 0.030631542 + - -0.07686653 + - 0.041529574 + - 0.00020558872 + - 0.06279164 + - -0.03607601 + - 0.04877371 + - 0.054333705 + - 0.12663308 + - -0.003843789 + - 0.00083064684 + - 0.069598824 + - 0.043968182 + - -0.03208848 + - -0.08519017 + - 0.0136859305 + - 0.022745186 + - -0.002784398 + - -0.0068232715 + - 0.037543006 + - 0.035281833 + - -0.06679111 + - 0.0215873 + - 0.03758603 + - -0.045440283 + - -0.051096193 + - -0.067934945 + - -0.03084997 + - -0.036311556 + - -0.014842255 + - -0.09372648 + - -0.031666476 + - 0.010247365 + - 0.015154026 + - -0.002330648 + - 0.024146315 + - -0.013367441 + - 0.006576458 + - 0.024498507 + - -0.1371471 + - 0.063926436 + - 0.1967554 + - -0.006042444 + - 0.053201348 + - -0.05518625 +- - 0.011566745 + - 0.025191609 + - -0.036694292 + - 0.05925345 + - -0.007202666 + - -0.041202188 + - 0.07718597 + - 0.037412453 + - 0.012524345 + - -0.006153889 + - 0.017013893 + - -0.07700467 + - -0.00042374025 + - 0.027820537 + - -0.015982956 + - -0.06823823 + - 0.0088968985 + - -0.020330165 + - -0.080409184 + - -0.013094234 + - -0.04111577 + - -0.02575625 + - -0.026604561 + - 0.033058524 + - -0.022127466 + - 0.021035336 + - -0.057914518 + - 0.033003185 + - 0.02975051 + - -0.062229656 + - 0.038802397 + - 0.03196761 + - 0.0153276855 + - 0.045363918 + - 0.05309402 + - 0.013351721 + - 0.041196864 + - 0.028114794 + - 0.019435482 + - -0.0032642381 + - -0.0035951673 + - -0.14286515 + - 0.03804074 + - -0.010979397 + - 0.02610546 + - 0.041429143 + - -0.016078437 + - 0.05356371 + - -0.05677031 + - 0.012202429 + - -0.03505195 + - -0.039740372 + - -0.046098847 + - -0.03911731 + - -0.018020066 + - 0.021650145 + - -0.0064859986 + - -0.026541203 + - 0.04864548 + - 0.04339141 + - 0.04626493 + - -0.03446706 + - -0.024146007 + - 0.055675633 + - 0.024611346 + - 0.019454936 + - 0.011295259 + - -0.025712365 + - -0.03224502 + - -0.032699518 + - -0.014605892 + - 0.0146714635 + - 0.010346884 + - 0.07415931 + - 0.080031686 + - -0.0042066616 + - 0.0026997656 + - -0.076720476 + - 0.05881217 + - -0.004803097 + - -0.087379314 + - -0.046413604 + - -0.039611273 + - 0.052492607 + - 0.024743188 + - 0.080671504 + - 0.1198378 + - 0.03179342 + - -0.11392431 + - 0.010694122 + - 0.01976001 + - 0.03866714 + - -0.034617126 + - -0.0061281016 + - -0.042578902 + - 0.022671003 + - 0.008379255 + - -0.019488879 + - -0.023300469 + - 0.24531509 + - 0.049447276 + - 0.029196257 + - -0.0057606567 + - -0.020439675 + - -0.07763772 + - -0.043004043 + - 0.0003997896 + - -0.066854045 + - 0.07072114 + - 0.0064844415 + - -0.058493868 + - -0.005078577 + - 0.030898178 + - 0.027523836 + - 0.027156036 + - -0.06816291 + - -0.06572996 + - 0.059115347 + - -0.031852506 + - 0.040637866 + - 0.057000894 + - 0.0069915373 + - 0.0062051546 + - -0.013209068 + - 0.031209558 + - -0.04034961 + - 0.070826106 + - -0.0000000000000000000000000000000048484194 + - 0.021934593 + - -0.10270966 + - 0.05622236 + - 0.09768862 + - -0.052761998 + - 0.019099213 + - -0.010433081 + - 0.07028384 + - -0.009163081 + - 0.059657086 + - 0.0093431 + - -0.015467531 + - -0.023266213 + - 0.023969134 + - 0.10212125 + - 0.09321899 + - -0.033179756 + - 0.011723363 + - -0.065606475 + - 0.031413328 + - -0.0237615 + - -0.0486851 + - 0.012292702 + - -0.0400642 + - -0.076479904 + - -0.035577532 + - -0.0013617877 + - -0.016482001 + - 0.016856715 + - -0.0004216753 + - -0.05272671 + - 0.029926106 + - -0.07283276 + - 0.06911143 + - -0.017620781 + - -0.005552996 + - 0.012905943 + - -0.022742389 + - 0.026754798 + - -0.025787508 + - -0.04020589 + - -0.013465992 + - 0.00076997397 + - 0.028548837 + - 0.031916283 + - -0.031540327 + - -0.029507546 + - -0.020180529 + - 0.048006866 + - -0.0013082745 + - -0.013749675 + - 0.020036789 + - -0.068734996 + - -0.021904256 + - -0.031283367 + - 0.049285103 + - 0.012069677 + - -0.058844592 + - -0.026464054 + - 0.05979334 + - 0.067720294 + - 0.034020975 + - -0.05295764 + - 0.059824925 + - -0.02547595 + - -0.020665556 + - -0.053789217 + - -0.09736991 + - 0.04800511 + - 0.05235638 + - -0.023362491 + - -0.06907576 + - 0.016694613 + - 0.02856358 + - -0.029173644 + - -0.03551531 + - -0.0126147205 + - 0.07328874 + - -0.019482115 + - -0.063127376 + - 0.096063904 + - -0.07734705 + - 0.015912902 + - -0.044809945 + - 0.016362015 + - -0.0007529947 + - -0.008698364 + - -0.098845914 + - 0.005773373 + - -0.071921706 + - -0.032704372 + - 0.019982614 + - 0.003827613 + - -0.025586382 + - 0.08240085 + - 0.0000000000000000000000000000000040850666 + - -0.029535603 + - 0.025522647 + - -0.051075805 + - 0.1553659 + - 0.0523394 + - -0.034587402 + - 0.13312408 + - -0.019211281 + - -0.0597844 + - 0.12298254 + - 0.010122296 + - -0.049751673 + - 0.058429115 + - 0.012729909 + - -0.016639234 + - 0.012746897 + - 0.04571337 + - -0.06986801 + - -0.04855961 + - -0.0049983296 + - -0.09038651 + - 0.069924004 + - 0.009439225 + - -0.0067679146 + - -0.106080666 + - 0.03100246 + - 0.04949134 + - -0.044834208 + - -0.007470489 + - -0.033606272 + - 0.07610607 + - 0.0071234847 + - -0.042222 + - 0.0708183 + - 0.047470734 + - 0.020847816 + - 0.15337874 + - -0.008404033 + - -0.025955822 + - 0.06075936 + - 0.06673509 + - 0.064682245 + - 0.049773723 + - 0.08875065 + - -0.0329716 + - 0.0703123 + - 0.017220959 + - -0.030130997 + - 0.03847389 + - 0.04841252 + - -0.060448505 + - 0.03055366 + - 0.015585768 + - -0.030456763 + - -0.009485868 + - -0.041138977 + - -0.06799712 + - 0.010231143 + - -0.025668688 + - 0.021700189 + - -0.06986792 + - 0.092408456 + - -0.03572349 + - 0.07020521 + - -0.06350947 + - -0.0328256 + - -0.046145674 + - 0.054110475 + - 0.051667217 + - 0.042987943 + - 0.013467606 + - 0.016568488 + - -0.04419887 + - -0.019691551 + - 0.03615324 + - -0.019701054 + - -0.1157151 + - 0.0059685465 + - 0.0045423866 + - -0.045008406 + - -0.06848648 + - -0.0853551 + - -0.07097772 + - 0.08044467 + - -0.057986956 + - 0.057813518 + - 0.050183795 + - 0.05942627 + - -0.036489185 + - 0.009364899 + - 0.052608218 + - 0.028058298 + - -0.033277407 + - -0.050756633 + - -0.01291192 + - -0.0000000142961065 + - -0.04050862 + - -0.085779026 + - 0.04517046 + - 0.021616641 + - -0.02237283 + - 0.012219842 + - -0.0324641 + - -0.016899578 + - -0.027103093 + - 0.005931705 + - 0.04030224 + - 0.026972173 + - -0.035654355 + - 0.07405438 + - 0.032423433 + - -0.090567395 + - -0.0316623 + - 0.040873803 + - -0.009927164 + - 0.030631542 + - -0.07686653 + - 0.041529574 + - 0.00020558872 + - 0.06279164 + - -0.03607601 + - 0.04877371 + - 0.054333705 + - 0.12663308 + - -0.003843789 + - 0.00083064684 + - 0.069598824 + - 0.043968182 + - -0.03208848 + - -0.08519017 + - 0.0136859305 + - 0.022745186 + - -0.002784398 + - -0.0068232715 + - 0.037543006 + - 0.035281833 + - -0.06679111 + - 0.0215873 + - 0.03758603 + - -0.045440283 + - -0.051096193 + - -0.067934945 + - -0.03084997 + - -0.036311556 + - -0.014842255 + - -0.09372648 + - -0.031666476 + - 0.010247365 + - 0.015154026 + - -0.002330648 + - 0.024146315 + - -0.013367441 + - 0.006576458 + - 0.024498507 + - -0.1371471 + - 0.063926436 + - 0.1967554 + - -0.006042444 + - 0.053201348 + - -0.05518625 +- - 0.011566745 + - 0.025191609 + - -0.036694292 + - 0.05925345 + - -0.007202666 + - -0.041202188 + - 0.07718597 + - 0.037412453 + - 0.012524345 + - -0.006153889 + - 0.017013893 + - -0.07700467 + - -0.00042374025 + - 0.027820537 + - -0.015982956 + - -0.06823823 + - 0.0088968985 + - -0.020330165 + - -0.080409184 + - -0.013094234 + - -0.04111577 + - -0.02575625 + - -0.026604561 + - 0.033058524 + - -0.022127466 + - 0.021035336 + - -0.057914518 + - 0.033003185 + - 0.02975051 + - -0.062229656 + - 0.038802397 + - 0.03196761 + - 0.0153276855 + - 0.045363918 + - 0.05309402 + - 0.013351721 + - 0.041196864 + - 0.028114794 + - 0.019435482 + - -0.0032642381 + - -0.0035951673 + - -0.14286515 + - 0.03804074 + - -0.010979397 + - 0.02610546 + - 0.041429143 + - -0.016078437 + - 0.05356371 + - -0.05677031 + - 0.012202429 + - -0.03505195 + - -0.039740372 + - -0.046098847 + - -0.03911731 + - -0.018020066 + - 0.021650145 + - -0.0064859986 + - -0.026541203 + - 0.04864548 + - 0.04339141 + - 0.04626493 + - -0.03446706 + - -0.024146007 + - 0.055675633 + - 0.024611346 + - 0.019454936 + - 0.011295259 + - -0.025712365 + - -0.03224502 + - -0.032699518 + - -0.014605892 + - 0.0146714635 + - 0.010346884 + - 0.07415931 + - 0.080031686 + - -0.0042066616 + - 0.0026997656 + - -0.076720476 + - 0.05881217 + - -0.004803097 + - -0.087379314 + - -0.046413604 + - -0.039611273 + - 0.052492607 + - 0.024743188 + - 0.080671504 + - 0.1198378 + - 0.03179342 + - -0.11392431 + - 0.010694122 + - 0.01976001 + - 0.03866714 + - -0.034617126 + - -0.0061281016 + - -0.042578902 + - 0.022671003 + - 0.008379255 + - -0.019488879 + - -0.023300469 + - 0.24531509 + - 0.049447276 + - 0.029196257 + - -0.0057606567 + - -0.020439675 + - -0.07763772 + - -0.043004043 + - 0.0003997896 + - -0.066854045 + - 0.07072114 + - 0.0064844415 + - -0.058493868 + - -0.005078577 + - 0.030898178 + - 0.027523836 + - 0.027156036 + - -0.06816291 + - -0.06572996 + - 0.059115347 + - -0.031852506 + - 0.040637866 + - 0.057000894 + - 0.0069915373 + - 0.0062051546 + - -0.013209068 + - 0.031209558 + - -0.04034961 + - 0.070826106 + - -0.0000000000000000000000000000000048484194 + - 0.021934593 + - -0.10270966 + - 0.05622236 + - 0.09768862 + - -0.052761998 + - 0.019099213 + - -0.010433081 + - 0.07028384 + - -0.009163081 + - 0.059657086 + - 0.0093431 + - -0.015467531 + - -0.023266213 + - 0.023969134 + - 0.10212125 + - 0.09321899 + - -0.033179756 + - 0.011723363 + - -0.065606475 + - 0.031413328 + - -0.0237615 + - -0.0486851 + - 0.012292702 + - -0.0400642 + - -0.076479904 + - -0.035577532 + - -0.0013617877 + - -0.016482001 + - 0.016856715 + - -0.0004216753 + - -0.05272671 + - 0.029926106 + - -0.07283276 + - 0.06911143 + - -0.017620781 + - -0.005552996 + - 0.012905943 + - -0.022742389 + - 0.026754798 + - -0.025787508 + - -0.04020589 + - -0.013465992 + - 0.00076997397 + - 0.028548837 + - 0.031916283 + - -0.031540327 + - -0.029507546 + - -0.020180529 + - 0.048006866 + - -0.0013082745 + - -0.013749675 + - 0.020036789 + - -0.068734996 + - -0.021904256 + - -0.031283367 + - 0.049285103 + - 0.012069677 + - -0.058844592 + - -0.026464054 + - 0.05979334 + - 0.067720294 + - 0.034020975 + - -0.05295764 + - 0.059824925 + - -0.02547595 + - -0.020665556 + - -0.053789217 + - -0.09736991 + - 0.04800511 + - 0.05235638 + - -0.023362491 + - -0.06907576 + - 0.016694613 + - 0.02856358 + - -0.029173644 + - -0.03551531 + - -0.0126147205 + - 0.07328874 + - -0.019482115 + - -0.063127376 + - 0.096063904 + - -0.07734705 + - 0.015912902 + - -0.044809945 + - 0.016362015 + - -0.0007529947 + - -0.008698364 + - -0.098845914 + - 0.005773373 + - -0.071921706 + - -0.032704372 + - 0.019982614 + - 0.003827613 + - -0.025586382 + - 0.08240085 + - 0.0000000000000000000000000000000040850666 + - -0.029535603 + - 0.025522647 + - -0.051075805 + - 0.1553659 + - 0.0523394 + - -0.034587402 + - 0.13312408 + - -0.019211281 + - -0.0597844 + - 0.12298254 + - 0.010122296 + - -0.049751673 + - 0.058429115 + - 0.012729909 + - -0.016639234 + - 0.012746897 + - 0.04571337 + - -0.06986801 + - -0.04855961 + - -0.0049983296 + - -0.09038651 + - 0.069924004 + - 0.009439225 + - -0.0067679146 + - -0.106080666 + - 0.03100246 + - 0.04949134 + - -0.044834208 + - -0.007470489 + - -0.033606272 + - 0.07610607 + - 0.0071234847 + - -0.042222 + - 0.0708183 + - 0.047470734 + - 0.020847816 + - 0.15337874 + - -0.008404033 + - -0.025955822 + - 0.06075936 + - 0.06673509 + - 0.064682245 + - 0.049773723 + - 0.08875065 + - -0.0329716 + - 0.0703123 + - 0.017220959 + - -0.030130997 + - 0.03847389 + - 0.04841252 + - -0.060448505 + - 0.03055366 + - 0.015585768 + - -0.030456763 + - -0.009485868 + - -0.041138977 + - -0.06799712 + - 0.010231143 + - -0.025668688 + - 0.021700189 + - -0.06986792 + - 0.092408456 + - -0.03572349 + - 0.07020521 + - -0.06350947 + - -0.0328256 + - -0.046145674 + - 0.054110475 + - 0.051667217 + - 0.042987943 + - 0.013467606 + - 0.016568488 + - -0.04419887 + - -0.019691551 + - 0.03615324 + - -0.019701054 + - -0.1157151 + - 0.0059685465 + - 0.0045423866 + - -0.045008406 + - -0.06848648 + - -0.0853551 + - -0.07097772 + - 0.08044467 + - -0.057986956 + - 0.057813518 + - 0.050183795 + - 0.05942627 + - -0.036489185 + - 0.009364899 + - 0.052608218 + - 0.028058298 + - -0.033277407 + - -0.050756633 + - -0.01291192 + - -0.0000000142961065 + - -0.04050862 + - -0.085779026 + - 0.04517046 + - 0.021616641 + - -0.02237283 + - 0.012219842 + - -0.0324641 + - -0.016899578 + - -0.027103093 + - 0.005931705 + - 0.04030224 + - 0.026972173 + - -0.035654355 + - 0.07405438 + - 0.032423433 + - -0.090567395 + - -0.0316623 + - 0.040873803 + - -0.009927164 + - 0.030631542 + - -0.07686653 + - 0.041529574 + - 0.00020558872 + - 0.06279164 + - -0.03607601 + - 0.04877371 + - 0.054333705 + - 0.12663308 + - -0.003843789 + - 0.00083064684 + - 0.069598824 + - 0.043968182 + - -0.03208848 + - -0.08519017 + - 0.0136859305 + - 0.022745186 + - -0.002784398 + - -0.0068232715 + - 0.037543006 + - 0.035281833 + - -0.06679111 + - 0.0215873 + - 0.03758603 + - -0.045440283 + - -0.051096193 + - -0.067934945 + - -0.03084997 + - -0.036311556 + - -0.014842255 + - -0.09372648 + - -0.031666476 + - 0.010247365 + - 0.015154026 + - -0.002330648 + - 0.024146315 + - -0.013367441 + - 0.006576458 + - 0.024498507 + - -0.1371471 + - 0.063926436 + - 0.1967554 + - -0.006042444 + - 0.053201348 + - -0.05518625 +- - 0.011566745 + - 0.025191609 + - -0.036694292 + - 0.05925345 + - -0.007202666 + - -0.041202188 + - 0.07718597 + - 0.037412453 + - 0.012524345 + - -0.006153889 + - 0.017013893 + - -0.07700467 + - -0.00042374025 + - 0.027820537 + - -0.015982956 + - -0.06823823 + - 0.0088968985 + - -0.020330165 + - -0.080409184 + - -0.013094234 + - -0.04111577 + - -0.02575625 + - -0.026604561 + - 0.033058524 + - -0.022127466 + - 0.021035336 + - -0.057914518 + - 0.033003185 + - 0.02975051 + - -0.062229656 + - 0.038802397 + - 0.03196761 + - 0.0153276855 + - 0.045363918 + - 0.05309402 + - 0.013351721 + - 0.041196864 + - 0.028114794 + - 0.019435482 + - -0.0032642381 + - -0.0035951673 + - -0.14286515 + - 0.03804074 + - -0.010979397 + - 0.02610546 + - 0.041429143 + - -0.016078437 + - 0.05356371 + - -0.05677031 + - 0.012202429 + - -0.03505195 + - -0.039740372 + - -0.046098847 + - -0.03911731 + - -0.018020066 + - 0.021650145 + - -0.0064859986 + - -0.026541203 + - 0.04864548 + - 0.04339141 + - 0.04626493 + - -0.03446706 + - -0.024146007 + - 0.055675633 + - 0.024611346 + - 0.019454936 + - 0.011295259 + - -0.025712365 + - -0.03224502 + - -0.032699518 + - -0.014605892 + - 0.0146714635 + - 0.010346884 + - 0.07415931 + - 0.080031686 + - -0.0042066616 + - 0.0026997656 + - -0.076720476 + - 0.05881217 + - -0.004803097 + - -0.087379314 + - -0.046413604 + - -0.039611273 + - 0.052492607 + - 0.024743188 + - 0.080671504 + - 0.1198378 + - 0.03179342 + - -0.11392431 + - 0.010694122 + - 0.01976001 + - 0.03866714 + - -0.034617126 + - -0.0061281016 + - -0.042578902 + - 0.022671003 + - 0.008379255 + - -0.019488879 + - -0.023300469 + - 0.24531509 + - 0.049447276 + - 0.029196257 + - -0.0057606567 + - -0.020439675 + - -0.07763772 + - -0.043004043 + - 0.0003997896 + - -0.066854045 + - 0.07072114 + - 0.0064844415 + - -0.058493868 + - -0.005078577 + - 0.030898178 + - 0.027523836 + - 0.027156036 + - -0.06816291 + - -0.06572996 + - 0.059115347 + - -0.031852506 + - 0.040637866 + - 0.057000894 + - 0.0069915373 + - 0.0062051546 + - -0.013209068 + - 0.031209558 + - -0.04034961 + - 0.070826106 + - -0.0000000000000000000000000000000048484194 + - 0.021934593 + - -0.10270966 + - 0.05622236 + - 0.09768862 + - -0.052761998 + - 0.019099213 + - -0.010433081 + - 0.07028384 + - -0.009163081 + - 0.059657086 + - 0.0093431 + - -0.015467531 + - -0.023266213 + - 0.023969134 + - 0.10212125 + - 0.09321899 + - -0.033179756 + - 0.011723363 + - -0.065606475 + - 0.031413328 + - -0.0237615 + - -0.0486851 + - 0.012292702 + - -0.0400642 + - -0.076479904 + - -0.035577532 + - -0.0013617877 + - -0.016482001 + - 0.016856715 + - -0.0004216753 + - -0.05272671 + - 0.029926106 + - -0.07283276 + - 0.06911143 + - -0.017620781 + - -0.005552996 + - 0.012905943 + - -0.022742389 + - 0.026754798 + - -0.025787508 + - -0.04020589 + - -0.013465992 + - 0.00076997397 + - 0.028548837 + - 0.031916283 + - -0.031540327 + - -0.029507546 + - -0.020180529 + - 0.048006866 + - -0.0013082745 + - -0.013749675 + - 0.020036789 + - -0.068734996 + - -0.021904256 + - -0.031283367 + - 0.049285103 + - 0.012069677 + - -0.058844592 + - -0.026464054 + - 0.05979334 + - 0.067720294 + - 0.034020975 + - -0.05295764 + - 0.059824925 + - -0.02547595 + - -0.020665556 + - -0.053789217 + - -0.09736991 + - 0.04800511 + - 0.05235638 + - -0.023362491 + - -0.06907576 + - 0.016694613 + - 0.02856358 + - -0.029173644 + - -0.03551531 + - -0.0126147205 + - 0.07328874 + - -0.019482115 + - -0.063127376 + - 0.096063904 + - -0.07734705 + - 0.015912902 + - -0.044809945 + - 0.016362015 + - -0.0007529947 + - -0.008698364 + - -0.098845914 + - 0.005773373 + - -0.071921706 + - -0.032704372 + - 0.019982614 + - 0.003827613 + - -0.025586382 + - 0.08240085 + - 0.0000000000000000000000000000000040850666 + - -0.029535603 + - 0.025522647 + - -0.051075805 + - 0.1553659 + - 0.0523394 + - -0.034587402 + - 0.13312408 + - -0.019211281 + - -0.0597844 + - 0.12298254 + - 0.010122296 + - -0.049751673 + - 0.058429115 + - 0.012729909 + - -0.016639234 + - 0.012746897 + - 0.04571337 + - -0.06986801 + - -0.04855961 + - -0.0049983296 + - -0.09038651 + - 0.069924004 + - 0.009439225 + - -0.0067679146 + - -0.106080666 + - 0.03100246 + - 0.04949134 + - -0.044834208 + - -0.007470489 + - -0.033606272 + - 0.07610607 + - 0.0071234847 + - -0.042222 + - 0.0708183 + - 0.047470734 + - 0.020847816 + - 0.15337874 + - -0.008404033 + - -0.025955822 + - 0.06075936 + - 0.06673509 + - 0.064682245 + - 0.049773723 + - 0.08875065 + - -0.0329716 + - 0.0703123 + - 0.017220959 + - -0.030130997 + - 0.03847389 + - 0.04841252 + - -0.060448505 + - 0.03055366 + - 0.015585768 + - -0.030456763 + - -0.009485868 + - -0.041138977 + - -0.06799712 + - 0.010231143 + - -0.025668688 + - 0.021700189 + - -0.06986792 + - 0.092408456 + - -0.03572349 + - 0.07020521 + - -0.06350947 + - -0.0328256 + - -0.046145674 + - 0.054110475 + - 0.051667217 + - 0.042987943 + - 0.013467606 + - 0.016568488 + - -0.04419887 + - -0.019691551 + - 0.03615324 + - -0.019701054 + - -0.1157151 + - 0.0059685465 + - 0.0045423866 + - -0.045008406 + - -0.06848648 + - -0.0853551 + - -0.07097772 + - 0.08044467 + - -0.057986956 + - 0.057813518 + - 0.050183795 + - 0.05942627 + - -0.036489185 + - 0.009364899 + - 0.052608218 + - 0.028058298 + - -0.033277407 + - -0.050756633 + - -0.01291192 + - -0.0000000142961065 + - -0.04050862 + - -0.085779026 + - 0.04517046 + - 0.021616641 + - -0.02237283 + - 0.012219842 + - -0.0324641 + - -0.016899578 + - -0.027103093 + - 0.005931705 + - 0.04030224 + - 0.026972173 + - -0.035654355 + - 0.07405438 + - 0.032423433 + - -0.090567395 + - -0.0316623 + - 0.040873803 + - -0.009927164 + - 0.030631542 + - -0.07686653 + - 0.041529574 + - 0.00020558872 + - 0.06279164 + - -0.03607601 + - 0.04877371 + - 0.054333705 + - 0.12663308 + - -0.003843789 + - 0.00083064684 + - 0.069598824 + - 0.043968182 + - -0.03208848 + - -0.08519017 + - 0.0136859305 + - 0.022745186 + - -0.002784398 + - -0.0068232715 + - 0.037543006 + - 0.035281833 + - -0.06679111 + - 0.0215873 + - 0.03758603 + - -0.045440283 + - -0.051096193 + - -0.067934945 + - -0.03084997 + - -0.036311556 + - -0.014842255 + - -0.09372648 + - -0.031666476 + - 0.010247365 + - 0.015154026 + - -0.002330648 + - 0.024146315 + - -0.013367441 + - 0.006576458 + - 0.024498507 + - -0.1371471 + - 0.063926436 + - 0.1967554 + - -0.006042444 + - 0.053201348 + - -0.05518625 +- - 0.011566745 + - 0.025191609 + - -0.036694292 + - 0.05925345 + - -0.007202666 + - -0.041202188 + - 0.07718597 + - 0.037412453 + - 0.012524345 + - -0.006153889 + - 0.017013893 + - -0.07700467 + - -0.00042374025 + - 0.027820537 + - -0.015982956 + - -0.06823823 + - 0.0088968985 + - -0.020330165 + - -0.080409184 + - -0.013094234 + - -0.04111577 + - -0.02575625 + - -0.026604561 + - 0.033058524 + - -0.022127466 + - 0.021035336 + - -0.057914518 + - 0.033003185 + - 0.02975051 + - -0.062229656 + - 0.038802397 + - 0.03196761 + - 0.0153276855 + - 0.045363918 + - 0.05309402 + - 0.013351721 + - 0.041196864 + - 0.028114794 + - 0.019435482 + - -0.0032642381 + - -0.0035951673 + - -0.14286515 + - 0.03804074 + - -0.010979397 + - 0.02610546 + - 0.041429143 + - -0.016078437 + - 0.05356371 + - -0.05677031 + - 0.012202429 + - -0.03505195 + - -0.039740372 + - -0.046098847 + - -0.03911731 + - -0.018020066 + - 0.021650145 + - -0.0064859986 + - -0.026541203 + - 0.04864548 + - 0.04339141 + - 0.04626493 + - -0.03446706 + - -0.024146007 + - 0.055675633 + - 0.024611346 + - 0.019454936 + - 0.011295259 + - -0.025712365 + - -0.03224502 + - -0.032699518 + - -0.014605892 + - 0.0146714635 + - 0.010346884 + - 0.07415931 + - 0.080031686 + - -0.0042066616 + - 0.0026997656 + - -0.076720476 + - 0.05881217 + - -0.004803097 + - -0.087379314 + - -0.046413604 + - -0.039611273 + - 0.052492607 + - 0.024743188 + - 0.080671504 + - 0.1198378 + - 0.03179342 + - -0.11392431 + - 0.010694122 + - 0.01976001 + - 0.03866714 + - -0.034617126 + - -0.0061281016 + - -0.042578902 + - 0.022671003 + - 0.008379255 + - -0.019488879 + - -0.023300469 + - 0.24531509 + - 0.049447276 + - 0.029196257 + - -0.0057606567 + - -0.020439675 + - -0.07763772 + - -0.043004043 + - 0.0003997896 + - -0.066854045 + - 0.07072114 + - 0.0064844415 + - -0.058493868 + - -0.005078577 + - 0.030898178 + - 0.027523836 + - 0.027156036 + - -0.06816291 + - -0.06572996 + - 0.059115347 + - -0.031852506 + - 0.040637866 + - 0.057000894 + - 0.0069915373 + - 0.0062051546 + - -0.013209068 + - 0.031209558 + - -0.04034961 + - 0.070826106 + - -0.0000000000000000000000000000000048484194 + - 0.021934593 + - -0.10270966 + - 0.05622236 + - 0.09768862 + - -0.052761998 + - 0.019099213 + - -0.010433081 + - 0.07028384 + - -0.009163081 + - 0.059657086 + - 0.0093431 + - -0.015467531 + - -0.023266213 + - 0.023969134 + - 0.10212125 + - 0.09321899 + - -0.033179756 + - 0.011723363 + - -0.065606475 + - 0.031413328 + - -0.0237615 + - -0.0486851 + - 0.012292702 + - -0.0400642 + - -0.076479904 + - -0.035577532 + - -0.0013617877 + - -0.016482001 + - 0.016856715 + - -0.0004216753 + - -0.05272671 + - 0.029926106 + - -0.07283276 + - 0.06911143 + - -0.017620781 + - -0.005552996 + - 0.012905943 + - -0.022742389 + - 0.026754798 + - -0.025787508 + - -0.04020589 + - -0.013465992 + - 0.00076997397 + - 0.028548837 + - 0.031916283 + - -0.031540327 + - -0.029507546 + - -0.020180529 + - 0.048006866 + - -0.0013082745 + - -0.013749675 + - 0.020036789 + - -0.068734996 + - -0.021904256 + - -0.031283367 + - 0.049285103 + - 0.012069677 + - -0.058844592 + - -0.026464054 + - 0.05979334 + - 0.067720294 + - 0.034020975 + - -0.05295764 + - 0.059824925 + - -0.02547595 + - -0.020665556 + - -0.053789217 + - -0.09736991 + - 0.04800511 + - 0.05235638 + - -0.023362491 + - -0.06907576 + - 0.016694613 + - 0.02856358 + - -0.029173644 + - -0.03551531 + - -0.0126147205 + - 0.07328874 + - -0.019482115 + - -0.063127376 + - 0.096063904 + - -0.07734705 + - 0.015912902 + - -0.044809945 + - 0.016362015 + - -0.0007529947 + - -0.008698364 + - -0.098845914 + - 0.005773373 + - -0.071921706 + - -0.032704372 + - 0.019982614 + - 0.003827613 + - -0.025586382 + - 0.08240085 + - 0.0000000000000000000000000000000040850666 + - -0.029535603 + - 0.025522647 + - -0.051075805 + - 0.1553659 + - 0.0523394 + - -0.034587402 + - 0.13312408 + - -0.019211281 + - -0.0597844 + - 0.12298254 + - 0.010122296 + - -0.049751673 + - 0.058429115 + - 0.012729909 + - -0.016639234 + - 0.012746897 + - 0.04571337 + - -0.06986801 + - -0.04855961 + - -0.0049983296 + - -0.09038651 + - 0.069924004 + - 0.009439225 + - -0.0067679146 + - -0.106080666 + - 0.03100246 + - 0.04949134 + - -0.044834208 + - -0.007470489 + - -0.033606272 + - 0.07610607 + - 0.0071234847 + - -0.042222 + - 0.0708183 + - 0.047470734 + - 0.020847816 + - 0.15337874 + - -0.008404033 + - -0.025955822 + - 0.06075936 + - 0.06673509 + - 0.064682245 + - 0.049773723 + - 0.08875065 + - -0.0329716 + - 0.0703123 + - 0.017220959 + - -0.030130997 + - 0.03847389 + - 0.04841252 + - -0.060448505 + - 0.03055366 + - 0.015585768 + - -0.030456763 + - -0.009485868 + - -0.041138977 + - -0.06799712 + - 0.010231143 + - -0.025668688 + - 0.021700189 + - -0.06986792 + - 0.092408456 + - -0.03572349 + - 0.07020521 + - -0.06350947 + - -0.0328256 + - -0.046145674 + - 0.054110475 + - 0.051667217 + - 0.042987943 + - 0.013467606 + - 0.016568488 + - -0.04419887 + - -0.019691551 + - 0.03615324 + - -0.019701054 + - -0.1157151 + - 0.0059685465 + - 0.0045423866 + - -0.045008406 + - -0.06848648 + - -0.0853551 + - -0.07097772 + - 0.08044467 + - -0.057986956 + - 0.057813518 + - 0.050183795 + - 0.05942627 + - -0.036489185 + - 0.009364899 + - 0.052608218 + - 0.028058298 + - -0.033277407 + - -0.050756633 + - -0.01291192 + - -0.0000000142961065 + - -0.04050862 + - -0.085779026 + - 0.04517046 + - 0.021616641 + - -0.02237283 + - 0.012219842 + - -0.0324641 + - -0.016899578 + - -0.027103093 + - 0.005931705 + - 0.04030224 + - 0.026972173 + - -0.035654355 + - 0.07405438 + - 0.032423433 + - -0.090567395 + - -0.0316623 + - 0.040873803 + - -0.009927164 + - 0.030631542 + - -0.07686653 + - 0.041529574 + - 0.00020558872 + - 0.06279164 + - -0.03607601 + - 0.04877371 + - 0.054333705 + - 0.12663308 + - -0.003843789 + - 0.00083064684 + - 0.069598824 + - 0.043968182 + - -0.03208848 + - -0.08519017 + - 0.0136859305 + - 0.022745186 + - -0.002784398 + - -0.0068232715 + - 0.037543006 + - 0.035281833 + - -0.06679111 + - 0.0215873 + - 0.03758603 + - -0.045440283 + - -0.051096193 + - -0.067934945 + - -0.03084997 + - -0.036311556 + - -0.014842255 + - -0.09372648 + - -0.031666476 + - 0.010247365 + - 0.015154026 + - -0.002330648 + - 0.024146315 + - -0.013367441 + - 0.006576458 + - 0.024498507 + - -0.1371471 + - 0.063926436 + - 0.1967554 + - -0.006042444 + - 0.053201348 + - -0.05518625 + diff --git a/router/tests/snapshots/test_http_embed__embeddings_single.snap b/router/tests/snapshots/test_http_embed__embeddings_single.snap new file mode 100644 index 00000000..dbc70364 --- /dev/null +++ b/router/tests/snapshots/test_http_embed__embeddings_single.snap @@ -0,0 +1,389 @@ +--- +source: router/tests/test_http_embed.rs +expression: embeddings_single +--- +- - 0.011566745 + - 0.025191609 + - -0.036694292 + - 0.05925345 + - -0.007202666 + - -0.041202188 + - 0.07718597 + - 0.037412453 + - 0.012524345 + - -0.006153889 + - 0.017013893 + - -0.07700467 + - -0.00042374025 + - 0.027820537 + - -0.015982956 + - -0.06823823 + - 0.0088968985 + - -0.020330165 + - -0.080409184 + - -0.013094234 + - -0.04111577 + - -0.02575625 + - -0.026604561 + - 0.033058524 + - -0.022127466 + - 0.021035336 + - -0.057914518 + - 0.033003185 + - 0.02975051 + - -0.062229656 + - 0.038802397 + - 0.03196761 + - 0.0153276855 + - 0.045363918 + - 0.05309402 + - 0.013351721 + - 0.041196864 + - 0.028114794 + - 0.019435482 + - -0.0032642381 + - -0.0035951673 + - -0.14286515 + - 0.03804074 + - -0.010979397 + - 0.02610546 + - 0.041429143 + - -0.016078437 + - 0.05356371 + - -0.05677031 + - 0.012202429 + - -0.03505195 + - -0.039740372 + - -0.046098847 + - -0.03911731 + - -0.018020066 + - 0.021650145 + - -0.0064859986 + - -0.026541203 + - 0.04864548 + - 0.04339141 + - 0.04626493 + - -0.03446706 + - -0.024146007 + - 0.055675633 + - 0.024611346 + - 0.019454936 + - 0.011295259 + - -0.025712365 + - -0.03224502 + - -0.032699518 + - -0.014605892 + - 0.0146714635 + - 0.010346884 + - 0.07415931 + - 0.080031686 + - -0.0042066616 + - 0.0026997656 + - -0.076720476 + - 0.05881217 + - -0.004803097 + - -0.087379314 + - -0.046413604 + - -0.039611273 + - 0.052492607 + - 0.024743188 + - 0.080671504 + - 0.1198378 + - 0.03179342 + - -0.11392431 + - 0.010694122 + - 0.01976001 + - 0.03866714 + - -0.034617126 + - -0.0061281016 + - -0.042578902 + - 0.022671003 + - 0.008379255 + - -0.019488879 + - -0.023300469 + - 0.24531509 + - 0.049447276 + - 0.029196257 + - -0.0057606567 + - -0.020439675 + - -0.07763772 + - -0.043004043 + - 0.0003997896 + - -0.066854045 + - 0.07072114 + - 0.0064844415 + - -0.058493868 + - -0.005078577 + - 0.030898178 + - 0.027523836 + - 0.027156036 + - -0.06816291 + - -0.06572996 + - 0.059115347 + - -0.031852506 + - 0.040637866 + - 0.057000894 + - 0.0069915373 + - 0.0062051546 + - -0.013209068 + - 0.031209558 + - -0.04034961 + - 0.070826106 + - -0.0000000000000000000000000000000048484194 + - 0.021934593 + - -0.10270966 + - 0.05622236 + - 0.09768862 + - -0.052761998 + - 0.019099213 + - -0.010433081 + - 0.07028384 + - -0.009163081 + - 0.059657086 + - 0.0093431 + - -0.015467531 + - -0.023266213 + - 0.023969134 + - 0.10212125 + - 0.09321899 + - -0.033179756 + - 0.011723363 + - -0.065606475 + - 0.031413328 + - -0.0237615 + - -0.0486851 + - 0.012292702 + - -0.0400642 + - -0.076479904 + - -0.035577532 + - -0.0013617877 + - -0.016482001 + - 0.016856715 + - -0.0004216753 + - -0.05272671 + - 0.029926106 + - -0.07283276 + - 0.06911143 + - -0.017620781 + - -0.005552996 + - 0.012905943 + - -0.022742389 + - 0.026754798 + - -0.025787508 + - -0.04020589 + - -0.013465992 + - 0.00076997397 + - 0.028548837 + - 0.031916283 + - -0.031540327 + - -0.029507546 + - -0.020180529 + - 0.048006866 + - -0.0013082745 + - -0.013749675 + - 0.020036789 + - -0.068734996 + - -0.021904256 + - -0.031283367 + - 0.049285103 + - 0.012069677 + - -0.058844592 + - -0.026464054 + - 0.05979334 + - 0.067720294 + - 0.034020975 + - -0.05295764 + - 0.059824925 + - -0.02547595 + - -0.020665556 + - -0.053789217 + - -0.09736991 + - 0.04800511 + - 0.05235638 + - -0.023362491 + - -0.06907576 + - 0.016694613 + - 0.02856358 + - -0.029173644 + - -0.03551531 + - -0.0126147205 + - 0.07328874 + - -0.019482115 + - -0.063127376 + - 0.096063904 + - -0.07734705 + - 0.015912902 + - -0.044809945 + - 0.016362015 + - -0.0007529947 + - -0.008698364 + - -0.098845914 + - 0.005773373 + - -0.071921706 + - -0.032704372 + - 0.019982614 + - 0.003827613 + - -0.025586382 + - 0.08240085 + - 0.0000000000000000000000000000000040850666 + - -0.029535603 + - 0.025522647 + - -0.051075805 + - 0.1553659 + - 0.0523394 + - -0.034587402 + - 0.13312408 + - -0.019211281 + - -0.0597844 + - 0.12298254 + - 0.010122296 + - -0.049751673 + - 0.058429115 + - 0.012729909 + - -0.016639234 + - 0.012746897 + - 0.04571337 + - -0.06986801 + - -0.04855961 + - -0.0049983296 + - -0.09038651 + - 0.069924004 + - 0.009439225 + - -0.0067679146 + - -0.106080666 + - 0.03100246 + - 0.04949134 + - -0.044834208 + - -0.007470489 + - -0.033606272 + - 0.07610607 + - 0.0071234847 + - -0.042222 + - 0.0708183 + - 0.047470734 + - 0.020847816 + - 0.15337874 + - -0.008404033 + - -0.025955822 + - 0.06075936 + - 0.06673509 + - 0.064682245 + - 0.049773723 + - 0.08875065 + - -0.0329716 + - 0.0703123 + - 0.017220959 + - -0.030130997 + - 0.03847389 + - 0.04841252 + - -0.060448505 + - 0.03055366 + - 0.015585768 + - -0.030456763 + - -0.009485868 + - -0.041138977 + - -0.06799712 + - 0.010231143 + - -0.025668688 + - 0.021700189 + - -0.06986792 + - 0.092408456 + - -0.03572349 + - 0.07020521 + - -0.06350947 + - -0.0328256 + - -0.046145674 + - 0.054110475 + - 0.051667217 + - 0.042987943 + - 0.013467606 + - 0.016568488 + - -0.04419887 + - -0.019691551 + - 0.03615324 + - -0.019701054 + - -0.1157151 + - 0.0059685465 + - 0.0045423866 + - -0.045008406 + - -0.06848648 + - -0.0853551 + - -0.07097772 + - 0.08044467 + - -0.057986956 + - 0.057813518 + - 0.050183795 + - 0.05942627 + - -0.036489185 + - 0.009364899 + - 0.052608218 + - 0.028058298 + - -0.033277407 + - -0.050756633 + - -0.01291192 + - -0.0000000142961065 + - -0.04050862 + - -0.085779026 + - 0.04517046 + - 0.021616641 + - -0.02237283 + - 0.012219842 + - -0.0324641 + - -0.016899578 + - -0.027103093 + - 0.005931705 + - 0.04030224 + - 0.026972173 + - -0.035654355 + - 0.07405438 + - 0.032423433 + - -0.090567395 + - -0.0316623 + - 0.040873803 + - -0.009927164 + - 0.030631542 + - -0.07686653 + - 0.041529574 + - 0.00020558872 + - 0.06279164 + - -0.03607601 + - 0.04877371 + - 0.054333705 + - 0.12663308 + - -0.003843789 + - 0.00083064684 + - 0.069598824 + - 0.043968182 + - -0.03208848 + - -0.08519017 + - 0.0136859305 + - 0.022745186 + - -0.002784398 + - -0.0068232715 + - 0.037543006 + - 0.035281833 + - -0.06679111 + - 0.0215873 + - 0.03758603 + - -0.045440283 + - -0.051096193 + - -0.067934945 + - -0.03084997 + - -0.036311556 + - -0.014842255 + - -0.09372648 + - -0.031666476 + - 0.010247365 + - 0.015154026 + - -0.002330648 + - 0.024146315 + - -0.013367441 + - 0.006576458 + - 0.024498507 + - -0.1371471 + - 0.063926436 + - 0.1967554 + - -0.006042444 + - 0.053201348 + - -0.05518625 + diff --git a/router/tests/snapshots/test_http_predict__predictions_batch.snap b/router/tests/snapshots/test_http_predict__predictions_batch.snap new file mode 100644 index 00000000..bd591761 --- /dev/null +++ b/router/tests/snapshots/test_http_predict__predictions_batch.snap @@ -0,0 +1,285 @@ +--- +source: router/tests/test_http_predict.rs +expression: predictions_batch +--- +- - score: 0.9972131 + label: neutral + - score: 0.0007965734 + label: approval + - score: 0.00029795468 + label: annoyance + - score: 0.00020120534 + label: admiration + - score: 0.0001964958 + label: realization + - score: 0.00013194357 + label: excitement + - score: 0.0001023708 + label: disappointment + - score: 0.000100379475 + label: disgust + - score: 0.00009976558 + label: anger + - score: 0.00008533838 + label: disapproval + - score: 0.00007990069 + label: joy + - score: 0.00007926623 + label: curiosity + - score: 0.000079192534 + label: amusement + - score: 0.000066899396 + label: confusion + - score: 0.00006555159 + label: optimism + - score: 0.000063790685 + label: love + - score: 0.00006372414 + label: sadness + - score: 0.000044774577 + label: fear + - score: 0.00004116887 + label: desire + - score: 0.000040867115 + label: surprise + - score: 0.000033375913 + label: caring + - score: 0.000027021175 + label: gratitude + - score: 0.000025252431 + label: embarrassment + - score: 0.000016195638 + label: pride + - score: 0.000013492125 + label: relief + - score: 0.000013144903 + label: grief + - score: 0.000011318783 + label: nervousness + - score: 0.000009880076 + label: remorse +- - score: 0.9972131 + label: neutral + - score: 0.0007965734 + label: approval + - score: 0.00029795468 + label: annoyance + - score: 0.00020120534 + label: admiration + - score: 0.0001964958 + label: realization + - score: 0.00013194357 + label: excitement + - score: 0.0001023708 + label: disappointment + - score: 0.000100379475 + label: disgust + - score: 0.00009976558 + label: anger + - score: 0.00008533838 + label: disapproval + - score: 0.00007990069 + label: joy + - score: 0.00007926623 + label: curiosity + - score: 0.000079192534 + label: amusement + - score: 0.000066899396 + label: confusion + - score: 0.00006555159 + label: optimism + - score: 0.000063790685 + label: love + - score: 0.00006372414 + label: sadness + - score: 0.000044774577 + label: fear + - score: 0.00004116887 + label: desire + - score: 0.000040867115 + label: surprise + - score: 0.000033375913 + label: caring + - score: 0.000027021175 + label: gratitude + - score: 0.000025252431 + label: embarrassment + - score: 0.000016195638 + label: pride + - score: 0.000013492125 + label: relief + - score: 0.000013144903 + label: grief + - score: 0.000011318783 + label: nervousness + - score: 0.000009880076 + label: remorse +- - score: 0.9972132 + label: neutral + - score: 0.0007965705 + label: approval + - score: 0.00029795355 + label: annoyance + - score: 0.00020120452 + label: admiration + - score: 0.00019649488 + label: realization + - score: 0.00013194351 + label: excitement + - score: 0.000102370424 + label: disappointment + - score: 0.000100379 + label: disgust + - score: 0.000099765304 + label: anger + - score: 0.000085338215 + label: disapproval + - score: 0.00007990062 + label: joy + - score: 0.00007926568 + label: curiosity + - score: 0.00007919242 + label: amusement + - score: 0.000066899396 + label: confusion + - score: 0.0000655515 + label: optimism + - score: 0.000063790605 + label: love + - score: 0.00006372412 + label: sadness + - score: 0.00004477469 + label: fear + - score: 0.000041168856 + label: desire + - score: 0.000040866962 + label: surprise + - score: 0.000033375803 + label: caring + - score: 0.000027021126 + label: gratitude + - score: 0.000025252299 + label: embarrassment + - score: 0.000016195578 + label: pride + - score: 0.000013492137 + label: relief + - score: 0.000013144917 + label: grief + - score: 0.000011318838 + label: nervousness + - score: 0.0000098800765 + label: remorse +- - score: 0.9972132 + label: neutral + - score: 0.0007965705 + label: approval + - score: 0.00029795355 + label: annoyance + - score: 0.00020120452 + label: admiration + - score: 0.00019649488 + label: realization + - score: 0.00013194351 + label: excitement + - score: 0.000102370424 + label: disappointment + - score: 0.000100379 + label: disgust + - score: 0.000099765304 + label: anger + - score: 0.000085338215 + label: disapproval + - score: 0.00007990062 + label: joy + - score: 0.00007926568 + label: curiosity + - score: 0.00007919242 + label: amusement + - score: 0.000066899396 + label: confusion + - score: 0.0000655515 + label: optimism + - score: 0.000063790605 + label: love + - score: 0.00006372412 + label: sadness + - score: 0.00004477469 + label: fear + - score: 0.000041168856 + label: desire + - score: 0.000040866962 + label: surprise + - score: 0.000033375803 + label: caring + - score: 0.000027021126 + label: gratitude + - score: 0.000025252299 + label: embarrassment + - score: 0.000016195578 + label: pride + - score: 0.000013492137 + label: relief + - score: 0.000013144917 + label: grief + - score: 0.000011318838 + label: nervousness + - score: 0.0000098800765 + label: remorse +- - score: 0.9972132 + label: neutral + - score: 0.0007965705 + label: approval + - score: 0.00029795355 + label: annoyance + - score: 0.00020120452 + label: admiration + - score: 0.00019649488 + label: realization + - score: 0.00013194351 + label: excitement + - score: 0.000102370424 + label: disappointment + - score: 0.000100379 + label: disgust + - score: 0.000099765304 + label: anger + - score: 0.000085338215 + label: disapproval + - score: 0.00007990062 + label: joy + - score: 0.00007926568 + label: curiosity + - score: 0.00007919242 + label: amusement + - score: 0.000066899396 + label: confusion + - score: 0.0000655515 + label: optimism + - score: 0.000063790605 + label: love + - score: 0.00006372412 + label: sadness + - score: 0.00004477469 + label: fear + - score: 0.000041168856 + label: desire + - score: 0.000040866962 + label: surprise + - score: 0.000033375803 + label: caring + - score: 0.000027021126 + label: gratitude + - score: 0.000025252299 + label: embarrassment + - score: 0.000016195578 + label: pride + - score: 0.000013492137 + label: relief + - score: 0.000013144917 + label: grief + - score: 0.000011318838 + label: nervousness + - score: 0.0000098800765 + label: remorse + diff --git a/router/tests/snapshots/test_http_predict__predictions_single.snap b/router/tests/snapshots/test_http_predict__predictions_single.snap new file mode 100644 index 00000000..b77f137d --- /dev/null +++ b/router/tests/snapshots/test_http_predict__predictions_single.snap @@ -0,0 +1,61 @@ +--- +source: router/tests/test_http_predict.rs +expression: predictions_single +--- +- score: 0.9972132 + label: neutral +- score: 0.0007965705 + label: approval +- score: 0.00029795355 + label: annoyance +- score: 0.00020120452 + label: admiration +- score: 0.00019649488 + label: realization +- score: 0.00013194351 + label: excitement +- score: 0.000102370424 + label: disappointment +- score: 0.000100379 + label: disgust +- score: 0.000099765304 + label: anger +- score: 0.000085338215 + label: disapproval +- score: 0.00007990062 + label: joy +- score: 0.00007926568 + label: curiosity +- score: 0.00007919242 + label: amusement +- score: 0.000066899396 + label: confusion +- score: 0.0000655515 + label: optimism +- score: 0.000063790605 + label: love +- score: 0.00006372412 + label: sadness +- score: 0.00004477469 + label: fear +- score: 0.000041168856 + label: desire +- score: 0.000040866962 + label: surprise +- score: 0.000033375803 + label: caring +- score: 0.000027021126 + label: gratitude +- score: 0.000025252299 + label: embarrassment +- score: 0.000016195578 + label: pride +- score: 0.000013492137 + label: relief +- score: 0.000013144917 + label: grief +- score: 0.000011318838 + label: nervousness +- score: 0.0000098800765 + label: remorse + diff --git a/router/tests/snapshots/test_http_rerank__ranks.snap b/router/tests/snapshots/test_http_rerank__ranks.snap new file mode 100644 index 00000000..d9be2aee --- /dev/null +++ b/router/tests/snapshots/test_http_rerank__ranks.snap @@ -0,0 +1,15 @@ +--- +source: router/tests/test_http_rerank.rs +assertion_line: 42 +expression: ranks +--- +- index: 2 + score: 0.9997739 + text: test +- index: 0 + score: 0.9997739 + text: test +- index: 1 + score: 0.3638598 + text: other + diff --git a/router/tests/test_http_embed.rs b/router/tests/test_http_embed.rs new file mode 100644 index 00000000..7c5db751 --- /dev/null +++ b/router/tests/test_http_embed.rs @@ -0,0 +1,53 @@ +mod common; + +use crate::common::{start_server, Score}; +use anyhow::Result; +use insta::internals::YamlMatcher; +use serde_json::json; +use text_embeddings_backend::DType; + +#[tokio::test] +#[cfg(feature = "http")] +async fn test_embeddings() -> Result<()> { + start_server( + "sentence-transformers/all-MiniLM-L6-v2".to_string(), + None, + DType::Float32, + ) + .await?; + + let request = json!({ + "inputs": "test" + }); + + let client = reqwest::Client::new(); + let res = client + .post("http://0.0.0.0:8090/embed") + .json(&request) + .send() + .await?; + + let embeddings_single = res.json::>>().await?; + let matcher = YamlMatcher::>>::new(); + insta::assert_yaml_snapshot!("embeddings_single", embeddings_single, &matcher); + + let request = json!({ + "inputs": vec!["test", "test", "test", "test", "test"], + }); + + let client = reqwest::Client::new(); + let res = client + .post("http://0.0.0.0:8090/embed") + .json(&request) + .send() + .await?; + + let embeddings_batch = res.json::>>().await?; + insta::assert_yaml_snapshot!("embeddings_batch", embeddings_batch, &matcher); + + for embeddings in &embeddings_batch { + assert_eq!(embeddings, &embeddings_single[0]); + } + + Ok(()) +} diff --git a/router/tests/test_http_predict.rs b/router/tests/test_http_predict.rs new file mode 100644 index 00000000..6869f721 --- /dev/null +++ b/router/tests/test_http_predict.rs @@ -0,0 +1,67 @@ +mod common; + +use crate::common::{start_server, Score}; +use anyhow::Result; +use insta::internals::YamlMatcher; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use text_embeddings_backend::DType; + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +pub struct SnapshotPrediction { + score: Score, + label: String, +} + +#[tokio::test] +#[cfg(feature = "http")] +async fn test_predict() -> Result<()> { + start_server( + "SamLowe/roberta-base-go_emotions".to_string(), + None, + DType::Float32, + ) + .await?; + + let request = json!({ + "inputs": "test" + }); + + let client = reqwest::Client::new(); + let res = client + .post("http://0.0.0.0:8090/predict") + .json(&request) + .send() + .await?; + + let predictions_single = res.json::>().await?; + let matcher = YamlMatcher::>::new(); + insta::assert_yaml_snapshot!("predictions_single", predictions_single, &matcher); + + let request = json!({ + "inputs": vec![ + vec!["test"], + vec!["test"], + vec!["test"], + vec!["test"], + vec!["test"], + ], + }); + + let client = reqwest::Client::new(); + let res = client + .post("http://0.0.0.0:8090/predict") + .json(&request) + .send() + .await?; + + let predictions_batch = res.json::>>().await?; + let matcher = YamlMatcher::>>::new(); + insta::assert_yaml_snapshot!("predictions_batch", predictions_batch, &matcher); + + for predictions in &predictions_batch { + assert_eq!(predictions, &predictions_single); + } + + Ok(()) +} diff --git a/router/tests/test_http_rerank.rs b/router/tests/test_http_rerank.rs new file mode 100644 index 00000000..0c5b6079 --- /dev/null +++ b/router/tests/test_http_rerank.rs @@ -0,0 +1,49 @@ +mod common; + +use crate::common::{start_server, Score}; +use anyhow::Result; +use insta::internals::YamlMatcher; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use text_embeddings_backend::DType; + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +pub struct SnapshotRank { + index: usize, + score: Score, + text: String, +} + +#[tokio::test] +#[cfg(feature = "http")] +async fn test_rerank() -> Result<()> { + start_server( + "BAAI/bge-reranker-base".to_string(), + Some("refs/pr/5".to_string()), + DType::Float32, + ) + .await?; + + let request = json!({ + "query": "test", + "texts": vec!["test", "other", "test"], + "return_text": true + }); + + let client = reqwest::Client::new(); + let res = client + .post("http://0.0.0.0:8090/rerank") + .json(&request) + .send() + .await?; + + let ranks = res.json::>().await?; + let matcher = YamlMatcher::>::new(); + insta::assert_yaml_snapshot!("ranks", ranks, &matcher); + + assert_eq!(ranks[0].index, 2); + assert_eq!(ranks[1].index, 0); + assert_eq!(ranks[0].score, ranks[1].score); + + Ok(()) +} From e4b01b425b7854f9327f982aa815503d51b7897f Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 30 Nov 2023 13:01:32 +0100 Subject: [PATCH 3/3] solve conflict --- router/src/lib.rs | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/router/src/lib.rs b/router/src/lib.rs index de3e902e..26a7a7a7 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -27,6 +27,7 @@ use text_embeddings_core::queue::Queue; use text_embeddings_core::tokenization::Tokenization; use text_embeddings_core::TextEmbeddingsError; use tokenizers::decoders::metaspace::PrependScheme; +use tokenizers::pre_tokenizers::sequence::Sequence; use tokenizers::{PreTokenizerWrapper, Tokenizer}; use tracing::Span; @@ -169,14 +170,33 @@ pub async fn run( m.set_prepend_scheme(PrependScheme::First); tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Metaspace(m)); } else if let PreTokenizerWrapper::Sequence(s) = pre_tokenizer { - // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer` - let mut s = s.clone(); - for pre_tokenizer in s.get_pre_tokenizers_mut() { - if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer { - m.set_prepend_scheme(PrependScheme::First); + let pre_tokenizers = s.get_pre_tokenizers(); + // Check if we have a Metaspace pre tokenizer in the sequence + let has_metaspace = pre_tokenizers + .iter() + .any(|t| matches!(t, PreTokenizerWrapper::Metaspace(_))); + + if has_metaspace { + let mut new_pre_tokenizers = Vec::with_capacity(s.get_pre_tokenizers().len()); + + for pre_tokenizer in pre_tokenizers { + if let PreTokenizerWrapper::WhitespaceSplit(_) = pre_tokenizer { + // Remove WhitespaceSplit + // This will be done by the Metaspace pre tokenizer + continue; + } + + let mut pre_tokenizer = pre_tokenizer.clone(); + + if let PreTokenizerWrapper::Metaspace(ref mut m) = pre_tokenizer { + m.set_prepend_scheme(PrependScheme::First); + } + new_pre_tokenizers.push(pre_tokenizer); } + tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Sequence(Sequence::new( + new_pre_tokenizers, + ))); } - tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Sequence(s)); } }