diff --git a/bindings/python/src/error.rs b/bindings/python/src/error.rs index d7f27e29b..7ba0d1256 100644 --- a/bindings/python/src/error.rs +++ b/bindings/python/src/error.rs @@ -23,6 +23,7 @@ impl std::error::Error for PyError {} pub struct ToPyResult(pub Result); impl std::convert::Into> for ToPyResult { fn into(self) -> PyResult { - self.0.map_err(|e| { exceptions::Exception::py_err(format!("{}", e)) }) + self.0 + .map_err(|e| exceptions::Exception::py_err(format!("{}", e))) } } diff --git a/bindings/python/src/models.rs b/bindings/python/src/models.rs index c7d3dd0e2..0fa616358 100644 --- a/bindings/python/src/models.rs +++ b/bindings/python/src/models.rs @@ -174,10 +174,7 @@ impl WordLevel { } } - match tk::models::wordlevel::WordLevel::from_files( - vocab, - unk_token, - ) { + match tk::models::wordlevel::WordLevel::from_files(vocab, unk_token) { Err(e) => { println!("Errors: {:?}", e); Err(exceptions::Exception::py_err( diff --git a/bindings/python/src/normalizers.rs b/bindings/python/src/normalizers.rs index 5b59b94a3..3e485c6e4 100644 --- a/bindings/python/src/normalizers.rs +++ b/bindings/python/src/normalizers.rs @@ -158,4 +158,3 @@ impl Strip { })) } } - diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs index f81333e56..5657fda00 100644 --- a/bindings/python/src/pre_tokenizers.rs +++ b/bindings/python/src/pre_tokenizers.rs @@ -91,13 +91,16 @@ pub struct CharDelimiterSplit {} impl CharDelimiterSplit { #[new] pub fn new(obj: &PyRawObject, delimiter: &str) -> PyResult<()> { - let chr_delimiter = delimiter.chars().nth(0).ok_or(exceptions::Exception::py_err( - "delimiter must be a single character", - ))?; - Ok(obj.init(PreTokenizer{ - pretok:Container::Owned(Box::new( - tk::pre_tokenizers::delimiter::CharDelimiterSplit::new(chr_delimiter) - )) + let chr_delimiter = delimiter + .chars() + .nth(0) + .ok_or(exceptions::Exception::py_err( + "delimiter must be a single character", + ))?; + Ok(obj.init(PreTokenizer { + pretok: Container::Owned(Box::new( + tk::pre_tokenizers::delimiter::CharDelimiterSplit::new(chr_delimiter), + )), })) } } diff --git a/bindings/python/src/processors.rs b/bindings/python/src/processors.rs index dfd1f123a..bd18c76fb 100644 --- a/bindings/python/src/processors.rs +++ b/bindings/python/src/processors.rs @@ -29,7 +29,6 @@ impl BertProcessing { } } - #[pyclass(extends=PostProcessor)] pub struct RobertaProcessing {} #[pymethods] diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index 81b029a89..a2eccdc8e 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -39,10 +39,10 @@ impl Tokenizer { } fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult { - Ok(self.tokenizer - .get_post_processor() - .map_or(0, |p| p.as_ref().added_tokens(is_pair)) - ) + Ok(self + .tokenizer + .get_post_processor() + .map_or(0, |p| p.as_ref().added_tokens(is_pair))) } #[args(kwargs = "**")] @@ -197,10 +197,11 @@ impl Tokenizer { } fn decode(&self, ids: Vec, skip_special_tokens: Option) -> PyResult { - ToPyResult(self.tokenizer.decode( - ids, - skip_special_tokens.unwrap_or(true), - )).into() + ToPyResult( + self.tokenizer + .decode(ids, skip_special_tokens.unwrap_or(true)), + ) + .into() } fn decode_batch( @@ -208,10 +209,11 @@ impl Tokenizer { sentences: Vec>, skip_special_tokens: Option, ) -> PyResult> { - ToPyResult(self.tokenizer.decode_batch( - sentences, - skip_special_tokens.unwrap_or(true), - )).into() + ToPyResult( + self.tokenizer + .decode_batch(sentences, skip_special_tokens.unwrap_or(true)), + ) + .into() } fn token_to_id(&self, token: &str) -> Option {