From 030b171ad0c3e2ea6a3819e8a59a45bee6b8476f Mon Sep 17 00:00:00 2001 From: "Felipe S. S. Schneider" Date: Mon, 17 Jul 2023 09:27:57 -0300 Subject: [PATCH] docs(src/lib.rs): refer to blog post --- README.md | 20 ++++++++++++-------- src/lib.rs | 26 +++++++++++++++++--------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 58a7b38..dcea867 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,18 @@ [Latest Version]: https://img.shields.io/crates/v/chat_splitter.svg [crates.io]: https://crates.io/crates/chat_splitter -Never exceed [OpenAI](https://openai.com/)'s [chat models](https://platform.openai.com/docs/api-reference/chat)' [maximum number of tokens](https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them) when using the [`async_openai`](https://github.com/64bit/async-openai) Rust crate. - -`chat-splitter` splits chats into 'outdated' and 'recent' messages. -You can split by -both -maximum message count and -maximum chat completion token count. -We use [`tiktoken_rs`](https://github.com/zurawiki/tiktoken-rs) for counting tokens. +> For more information, +> please refer to the [blog announcement](https://schneiderfelipe.github.io/posts/chat-splitter-first-release/). + +When utilizing the [`async_openai`](https://github.com/64bit/async-openai) [Rust](https://www.rust-lang.org/) crate, +it is crucial to ensure that you do not exceed +the [maximum number of tokens](https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them) specified by [OpenAI](https://openai.com/)'s [chat models](https://platform.openai.com/docs/api-reference/chat). + +[`chat-splitter`](https://crates.io/crates/chat_splitter) categorizes chat messages into 'outdated' and 'recent' messages, +allowing you to split them based on both the maximum +message count and the maximum chat completion token count. +The token counting functionality is provided by +[`tiktoken_rs`](https://github.com/zurawiki/tiktoken-rs). ## Usage diff --git a/src/lib.rs b/src/lib.rs index eaefee0..30f0732 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,14 +5,18 @@ //! [Latest Version]: https://img.shields.io/crates/v/chat_splitter.svg //! [crates.io]: https://crates.io/crates/chat_splitter //! -//! Never exceed [OpenAI](https://openai.com/)'s [chat models](https://platform.openai.com/docs/api-reference/chat)' [maximum number of tokens](https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them) when using the [`async_openai`](https://github.com/64bit/async-openai) Rust crate. +//! > For more information, +//! > please refer to the [blog announcement](https://schneiderfelipe.github.io/posts/chat-splitter-first-release/). //! -//! `chat-splitter` splits chats into 'outdated' and 'recent' messages. -//! You can split by -//! both -//! maximum message count and -//! maximum chat completion token count. -//! We use [`tiktoken_rs`](https://github.com/zurawiki/tiktoken-rs) for counting tokens. +//! When utilizing the [`async_openai`](https://github.com/64bit/async-openai) [Rust](https://www.rust-lang.org/) crate, +//! it is crucial to ensure that you do not exceed +//! the [maximum number of tokens](https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them) specified by [OpenAI](https://openai.com/)'s [chat models](https://platform.openai.com/docs/api-reference/chat). +//! +//! [`chat-splitter`](https://crates.io/crates/chat_splitter) categorizes chat messages into 'outdated' and 'recent' messages, +//! allowing you to split them based on both the maximum +//! message count and the maximum chat completion token count. +//! The token counting functionality is provided by +//! [`tiktoken_rs`](https://github.com/zurawiki/tiktoken-rs). //! //! # Usage //! @@ -51,7 +55,9 @@ use tiktoken_rs::model::get_context_size; /// see the [crate documentation](`crate`). #[derive(Clone, Debug)] pub struct ChatSplitter { - /// The model to use for tokenization, e.g., `gpt-3.5-turbo`. + /// The model to use for tokenization, + /// e.g., + /// `gpt-3.5-turbo`. /// /// It is passed to [`tiktoken_rs`] to select the correct tokenizer. model: String, @@ -149,7 +155,9 @@ impl ChatSplitter { self } - /// Set the model to use for tokenization, e.g., `gpt-3.5-turbo`. + /// Set the model to use for tokenization, + /// e.g., + /// `gpt-3.5-turbo`. /// /// It is passed to [`tiktoken_rs`] to select the correct tokenizer. #[inline]