add opennlp

dainiusjocas · Aug 10, 2023 · edd9c8c · edd9c8c
1 parent c705745
commit edd9c8c
Show file tree

Hide file tree

Showing 7 changed files with 24 additions and 16 deletions.
diff --git a/deps.edn b/deps.edn
@@ -1,20 +1,21 @@
 {:paths
  ["src" "resources"]
  :deps
- {org.clojure/clojure                      {:mvn/version "1.11.1"}
-  org.clojure/tools.cli                    {:mvn/version "1.0.219"}
-  org.apache.lucene/lucene-core            {:mvn/version "9.7.0"}
-  org.apache.lucene/lucene-analysis-common {:mvn/version "9.7.0"}
-  org.apache.lucene/lucene-monitor         {:mvn/version "9.7.0"}
-  org.apache.lucene/lucene-queries         {:mvn/version "9.7.0"}
-  org.apache.lucene/lucene-queryparser     {:mvn/version "9.7.0"}
-  metosin/jsonista                         {:mvn/version "0.3.7"}
-  lt.jocas/lucene-custom-analyzer          {:mvn/version "1.0.28"}
-  lmgrep/lucene-monitor-helpers            {:local/root "modules/lucene-monitor-helpers"}
-  lt.jocas/lucene-query-parsing            {:mvn/version "1.0.8"}
-  lt.jocas/lucene-text-analysis            {:mvn/version "1.0.21"}
-  babashka/fs                              {:mvn/version "0.4.19"}
-  com.github.clj-easy/graal-build-time     {:mvn/version "1.0.5"}}
+ {org.clojure/clojure                       {:mvn/version "1.11.1"}
+  org.clojure/tools.cli                     {:mvn/version "1.0.219"}
+  org.apache.lucene/lucene-core             {:mvn/version "9.7.0"}
+  org.apache.lucene/lucene-analysis-common  {:mvn/version "9.7.0"}
+  org.apache.lucene/lucene-analysis-opennlp {:mvn/version "9.7.0"}
+  org.apache.lucene/lucene-monitor          {:mvn/version "9.7.0"}
+  org.apache.lucene/lucene-queries          {:mvn/version "9.7.0"}
+  org.apache.lucene/lucene-queryparser      {:mvn/version "9.7.0"}
+  metosin/jsonista                          {:mvn/version "0.3.7"}
+  lt.jocas/lucene-custom-analyzer           {:mvn/version "1.0.28"}
+  lmgrep/lucene-monitor-helpers             {:local/root "modules/lucene-monitor-helpers"}
+  lt.jocas/lucene-query-parsing             {:mvn/version "1.0.8"}
+  lt.jocas/lucene-text-analysis             {:mvn/version "1.0.21"}
+  babashka/fs                               {:mvn/version "0.4.19"}
+  com.github.clj-easy/graal-build-time      {:mvn/version "1.0.5"}}
  :aliases
  {:dev
   {:extra-paths ["dev" "test" "test/resources"]

diff --git a/test/lmgrep/lucene/analysis_test.clj b/test/lmgrep/lucene/analysis_test.clj
@@ -155,7 +155,9 @@
         tokenizer-names (keys components)
         args {"simplepatternsplit" {"pattern" " "}
               "simplepattern"      {"pattern" " "}
-              "pattern"            {"pattern" " "}}]
+              "pattern"            {"pattern" " "}
+              "opennlp"            {"sentenceModel" "test/resources/opennlp-en-ud-ewt-sentence-1.0-1.9.3.bin"
+                                    "tokenizerModel" "test/resources/opennlp-en-ud-ewt-tokens-1.0-1.9.3.bin"}}]
     (is (seq tokenizer-names))
     (doseq [tokenizer-name tokenizer-names]
       (try
@@ -201,7 +203,10 @@
               "hunspellstem"            {"dictionary" "test/resources/hunspell_dict.dic"
                                          "affix"      "test/resources/hunspell_dict.aff"}
               "dropifflagged"           {"dropFlags" "2"}
-              "word2vecsynonym"         {"model" "test/resources/word2vec-model.zip"}}]
+              "word2vecsynonym"         {"model" "test/resources/word2vec-model.zip"}
+              "opennlppos"              {"posTaggerModel" "test/resources/opennlp-en-ud-ewt-pos-1.0-1.9.3.bin"}
+              "opennlplemmatizer"       {"dictionary" "test/resources/en-lemmatizer.dict"}
+              "opennlpchunker"          {"chunkerModel" "test/resources/en-chunker.bin"}}]
     (is (seq token-filter-names))
     (doseq [token-filter-name token-filter-names]
       (try

diff --git a/test/resources/en-chunker.bin b/test/resources/en-chunker.bin
diff --git a/test/resources/en-lemmatizer.dict b/test/resources/en-lemmatizer.dict
@@ -0,0 +1,2 @@
+AWOL	JJ	AWOL
+Aaronic	JJ	Aaronic
diff --git a/test/resources/opennlp-en-ud-ewt-pos-1.0-1.9.3.bin b/test/resources/opennlp-en-ud-ewt-pos-1.0-1.9.3.bin
diff --git a/test/resources/opennlp-en-ud-ewt-sentence-1.0-1.9.3.bin b/test/resources/opennlp-en-ud-ewt-sentence-1.0-1.9.3.bin
diff --git a/test/resources/opennlp-en-ud-ewt-tokens-1.0-1.9.3.bin b/test/resources/opennlp-en-ud-ewt-tokens-1.0-1.9.3.bin