> toks <- tokens("a x c y d") %>%
+ tokens_remove(c("x", "y"), padding = TRUE)
> toks
tokens from 1 document.
text1 :
[1] "a" "" "c" "" "d"
> tokens_lookup(toks, dictionary(list("aa" = "a", "bb" = "b")))
tokens from 1 document.
text1 :
[1] "aa"
> tokens_lookup(toks, dictionary(list("aa" = "a", "bb" = "b")), exclusive = FALSE)
tokens from 1 document.
text1 :
[1] "AA" "AA" "c" "AA" "d"