From a71e27633c8a2ff36838d209a8f6bf401ce87af1 Mon Sep 17 00:00:00 2001 From: Roger Zurawicki Date: Tue, 14 Mar 2023 00:09:29 -0400 Subject: [PATCH] chore: Update dependencies in Cargo.toml - Update dependencies for better stability and security - Upgrade tiktoken-rs to version 0.2.1 for improved performance and compatibility - Overall improvements and fixes to cargo.toml file. --- Cargo.lock | 106 +++++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 10 ++--- src/llms/openai.rs | 3 +- 3 files changed, 104 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dfa11e6..777c473 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,9 +134,9 @@ dependencies = [ [[package]] name = "async-openai" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb2cb6b01a5d4171ff51a1f72d93be91a6eedf8a316368567bde63533cd7514" +checksum = "25a497fb330310be352a9e30a040804cd3f16f5ae2e57eeedd50c0f530189c88" dependencies = [ "backoff", "base64 0.21.0", @@ -893,7 +893,7 @@ dependencies = [ "tera", "tiktoken-rs", "tokio", - "toml 0.7.2", + "toml 0.7.3", "toml_edit", "which", ] @@ -1083,6 +1083,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indoc" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" + [[package]] name = "instant" version = "0.1.12" @@ -1205,6 +1211,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -1520,6 +1535,66 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pyo3" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06a3d8e8a46ab2738109347433cb7b96dffda2e4a218b03ef27090238886b147" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75439f995d07ddfad42b192dfcf3bc66a7ecfd8b4a1f5f6f046aa5c2c5d7677d" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839526a5c07a17ff44823679b68add4a58004de00512a95b6c1c98a6dcac0ee5" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd44cf207476c6a9760c4653559be4f206efafb924d3e4cbf2721475fc0d6cc5" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1f43d8e30460f36350d18631ccf85ded64c059829208fe680904c65bcd0a4c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quote" version = "1.0.23" @@ -1962,6 +2037,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "target-lexicon" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" + [[package]] name = "tempfile" version = "3.4.0" @@ -2032,9 +2113,9 @@ dependencies = [ [[package]] name = "tiktoken-rs" -version = "0.1.4" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4918eef4e8d3e45bfab9eb145082ee12a20a6cc1b352db525e9ad4c453dfb87" +checksum = "1173648da6a43824f571d3a14f601e311264af34edea71953b558bfeb3dfe59d" dependencies = [ "anyhow", "base64 0.21.0", @@ -2042,6 +2123,7 @@ dependencies = [ "fancy-regex", "lazy_static", "parking_lot", + "pyo3", "rustc-hash", ] @@ -2177,9 +2259,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6" +checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" dependencies = [ "serde", "serde_spanned", @@ -2198,9 +2280,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.5" +version = "0.19.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7082a95d48029677a28f181e5f6422d0c8339ad8396a39d3f33d62a90c1f6c30" +checksum = "08de71aa0d6e348f070457f85af8bd566e2bc452156a423ddf22861b3a953fae" dependencies = [ "indexmap", "serde", @@ -2345,6 +2427,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unindent" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" + [[package]] name = "untrusted" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index a45a2ef..ca2d9d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ path = "src/main.rs" [dependencies] anyhow = "1.0.69" -async-openai = "0.9.2" +async-openai = "0.9.4" async-std = "1.12.0" async-trait = "0.1.66" backoff = "0.4.0" @@ -31,16 +31,16 @@ log = "0.4.17" regex = "1.7.1" reqwest = { version = "0.11.14", features = ["json", "gzip", "brotli", "rustls-tls"] } serde = { version = "1.0", features = ["derive"] } -serde_derive = "1.0.152" +serde_derive = "1.0.155" serde_json = "1.0.94" simple_logger = "4.0.0" strum = "0.24.1" strum_macros = "0.24.3" tera = { version = "1.18.0", default-features = false } -tiktoken-rs = "0.1.4" +tiktoken-rs = "0.2.1" tokio = { version = "1.26.0", features = ["full"] } -toml = "0.7.2" -toml_edit = "0.19.4" +toml = "0.7.3" +toml_edit = "0.19.6" which = "4.4.0" [dev-dependencies] diff --git a/src/llms/openai.rs b/src/llms/openai.rs index 53dccba..0618559 100644 --- a/src/llms/openai.rs +++ b/src/llms/openai.rs @@ -2,7 +2,7 @@ use anyhow::{anyhow, bail, Result}; use async_trait::async_trait; -use tiktoken_rs::tiktoken::{p50k_base, CoreBPE}; +use tiktoken_rs::{p50k_base, CoreBPE}; use crate::settings::OpenAISettings; use async_openai::{ @@ -126,6 +126,7 @@ impl LlmClient for OpenAIClient { lazy_static! { static ref BPE_TOKENIZER: CoreBPE = p50k_base().unwrap(); } + // TODO adjust token counting let n_tokens = 100; let tokens = BPE_TOKENIZER.encode_with_special_tokens(prompt);