exclude empty pos tags in python, lint

bminixhofer · Jan 13, 2021 · e76bed4 · e76bed4
1 parent 54ce8be
commit e76bed4
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 7 deletions.
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
@@ -361,7 +361,14 @@ impl PyToken {
             .word
             .tags
             .iter()
-            .map(|x| self.tagger.id_to_tag(x.pos_id))
+            .filter_map(|x| {
+                let pos = self.tagger.id_to_tag(x.pos_id);
+                if pos.is_empty() {
+                    None
+                } else {
+                    Some(pos)
+                }
+            })
             .collect();
         tags.sort_unstable();
         tags.dedup();

diff --git a/nlprule/Cargo.toml b/nlprule/Cargo.toml
@@ -27,16 +27,16 @@ rayon-cond = "0.1.0"
 rayon = "1.5"
 
 clap = { version = "3.0.0-beta.1", optional = true }
-env_logger = { version = "0.8.1", optional = true }
+env_logger = { version = "0.8", optional = true }
 
-serde-xml-rs = { git = "https://github.com/RReverser/serde-xml-rs/", optional = true }
+serde-xml-rs = { version = "0.4", optional = true }
 xml-rs = { version = "0.8.3", optional = true }
 roxmltree = { version = "0.14.0", optional = true }
 serde_json = { version = "1", optional = true }
 
 [dev-dependencies]
-quickcheck = "0.9"
-quickcheck_macros = "0.9"
+quickcheck = "1.0"
+quickcheck_macros = "1.0"
 
 [features]
 compile = ["serde-xml-rs", "xml-rs", "roxmltree", "serde_json"]

diff --git a/nlprule/src/rule/from_structure/mod.rs b/nlprule/src/rule/from_structure/mod.rs
@@ -821,7 +821,7 @@ fn parse_pos_filter(postag: &str, postag_regexp: Option<&str>, tagger: &Tagger)
             tagger,
         )),
         Some(_) | None => POSFilter::new(PosMatcher::new(
-            Matcher::new_string(either::Left(postag.to_string().into()), false, false, true),
+            Matcher::new_string(either::Left(postag.into()), false, false, true),
             tagger,
         )),
     }

diff --git a/nlprule/src/tokenizer.rs b/nlprule/src/tokenizer.rs
@@ -70,7 +70,7 @@ fn get_token_strs(text: &str) -> Vec<&str> {
 
 /// *Finalizes* the tokens by e. g. adding a specific UNKNOWN part-of-speech tag.
 /// After finalization grammatical error correction rules can be used on the tokens.
-pub fn finalize<'t>(tokens: Vec<IncompleteToken<'t>>) -> Vec<Token<'t>> {
+pub fn finalize(tokens: Vec<IncompleteToken>) -> Vec<Token> {
     if tokens.is_empty() {
         return Vec::new();
     }