fix usage of parameters

kermitt2 · Feb 6, 2024 · 8282dad · 8282dad
1 parent 5750ad7
commit 8282dad
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 7 deletions.
diff --git a/grobid-core/src/main/java/org/grobid/core/lexicon/Lexicon.java b/grobid-core/src/main/java/org/grobid/core/lexicon/Lexicon.java
@@ -566,7 +566,7 @@ else if (code.equals("it"))
             return "Italian";
         else if (code.equals("jp"))
             return "Japanese";
-        else if (code.equals("kr"))
+        else if (code.equals("kr") || code.equals("ko"))
             return "Korean";
         else if (code.equals("nl"))
             return "Deutch";

diff --git a/grobid-trainer/src/main/java/org/grobid/trainer/PatentParserTrainer.java b/grobid-trainer/src/main/java/org/grobid/trainer/PatentParserTrainer.java
@@ -182,16 +182,22 @@ public boolean accept(File dir, String name) {
                         List<List<LayoutToken>> segmentedAccumulatedTokens = new ArrayList<>();
                         List<List<String>> segmentedAccumulatedLabels = new ArrayList<>();
 
-                        if (accumulatedTokens.size() > 1000) {                         
+                        int maxSequence = 1000;
+                        if (GrobidProperties.getGrobidCRFEngineName("patent-citation").equals("delft")) {
+                            List<String> newTexts = new ArrayList<>();
+                            maxSequence = GrobidProperties.getDelftTrainingMaxSequenceLength("patent-citation");
+                        }
+
+                        if (accumulatedTokens.size() > maxSequence) {                         
                             // we have a problem of sequence length for Deep Learning algorithms
                             // we need to segment further. We ensure here that we don't segment 
                             // near or inside patent or NPL references 
                             int k = 0; 
                             while(k<accumulatedTokens.size()) {
                                 int origin = k;
 
-                                if (k+1000 < accumulatedTokens.size()) {
-                                    k = k+1000;
+                                if (k+maxSequence < accumulatedTokens.size()) {
+                                    k = k+maxSequence;
                                     // adjust position to avoid reference label
                                     while (accumulatedLabels.get(k-1).endsWith("refNPL>") || accumulatedLabels.get(k-1).endsWith("refPatent>")) {
                                         k--;
@@ -355,16 +361,22 @@ public void createDataSet(String setName, String corpusPath, String outputPath,
                                     List<List<LayoutToken>> segmentedAccumulatedTokens = new ArrayList<>();
                                     List<List<String>> segmentedAccumulatedLabels = new ArrayList<>();
 
-                                    if (accumulatedTokens.size() > 1000) {
+                                    int maxSequence = 1000;
+                                    if (GrobidProperties.getGrobidCRFEngineName("patent-citation").equals("delft")) {
+                                        List<String> newTexts = new ArrayList<>();
+                                        maxSequence = GrobidProperties.getDelftTrainingMaxSequenceLength("patent-citation");
+                                    }
+
+                                    if (accumulatedTokens.size() > maxSequence) {
                                         // we have a problem of sequence length for Deep Learning algorithms
                                         // we need to segment further. We ensure here that we don't segment 
                                         // near or inside patent or NPL references 
                                         int k = 0; 
                                         while(k<accumulatedTokens.size()) {
                                             int origin = k;
 
-                                            if (k+1000 < accumulatedTokens.size()) {
-                                                k = k+1000;
+                                            if (k+maxSequence < accumulatedTokens.size()) {
+                                                k = k+maxSequence;
                                                 // adjust position to avoid reference label
                                                 while (accumulatedLabels.get(k-1).endsWith("refNPL>") || accumulatedLabels.get(k-1).endsWith("refPatent>")) {
                                                     k--;