Merge adc0a94 into b0560b5

kermitt2 · Oct 12, 2020 · e259768 · e259768
2 parents b0560b5 + adc0a94
commit e259768
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 13 deletions.
diff --git a/grobid-core/src/main/java/org/grobid/core/main/LibraryLoader.java b/grobid-core/src/main/java/org/grobid/core/main/LibraryLoader.java
@@ -1,5 +1,6 @@
 package org.grobid.core.main;
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.SystemUtils;
 import org.grobid.core.engines.tagging.GrobidCRFEngine;
@@ -11,12 +12,13 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.io.FileFilter;
 import java.io.FilenameFilter;
 import java.lang.reflect.Field;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
 
 import static org.apache.commons.lang3.ArrayUtils.isEmpty;
 
@@ -95,8 +97,9 @@ public static void load() {
 
             }
 
-            if (GrobidProperties.getGrobidCRFEngine() == GrobidCRFEngine.WAPITI ||
-                GrobidProperties.getGrobidCRFEngine() == GrobidCRFEngine.DELFT) {
+            Set<GrobidCRFEngine> distinctModels = GrobidProperties.getDistinctModels();
+            if (CollectionUtils
+                .containsAny(distinctModels, Arrays.asList(GrobidCRFEngine.WAPITI, GrobidCRFEngine.DELFT))) {
                 // note: if DeLFT is used, we still make Wapiti available for models not existing in DeLFT (currently segmentation and 
                 // fulltext)
                 File[] wapitiLibFiles = libraryFolder.listFiles(new FilenameFilter() {
@@ -110,7 +113,7 @@ public boolean accept(File dir, String name) {
                     LOGGER.info("No wapiti library in the Grobid home folder");
                 } else {
                     LOGGER.info("Loading Wapiti native library...");
-                    if (GrobidProperties.getGrobidCRFEngine() == GrobidCRFEngine.DELFT) {
+                    if (CollectionUtils.containsAny(distinctModels, Collections.singletonList(GrobidCRFEngine.DELFT))) {
                         // if DeLFT will be used, we must not load libstdc++, it would create a conflict with tensorflow libstdc++ version
                         // so we temporary rename the lib so that it is not loaded in this case
                         // note that we know that, in this case, the local lib can be ignored because as DeFLT and tensorflow are installed
@@ -126,7 +129,7 @@ public boolean accept(File dir, String name) {
                     try {
                         System.load(wapitiLibFiles[0].getAbsolutePath());
                     } finally {
-                        if (GrobidProperties.getGrobidCRFEngine() == GrobidCRFEngine.DELFT) {
+                        if (CollectionUtils.containsAny(distinctModels, Arrays.asList(GrobidCRFEngine.DELFT))) {
                             // restore libstdc++
                             String libstdcppPathNew = libraryFolder.getAbsolutePath() + File.separator + "libstdc++.so.6.new";
                             File libstdcppFileNew = new File(libstdcppPathNew);
@@ -140,7 +143,7 @@ public boolean accept(File dir, String name) {
             }
 
 
-            if (GrobidProperties.getGrobidCRFEngine() == GrobidCRFEngine.DELFT) {
+            if (CollectionUtils.containsAny(distinctModels, Collections.singletonList(GrobidCRFEngine.DELFT))) {
                 LOGGER.info("Loading JEP native library for DeLFT... " + libraryFolder.getAbsolutePath());
                 // actual loading will be made at JEP initialization, so we just need to add the path in the 
                 // java.library.path (JEP will anyway try to load from java.library.path, so explicit file 

diff --git a/grobid-core/src/main/java/org/grobid/core/utilities/GrobidProperties.java b/grobid-core/src/main/java/org/grobid/core/utilities/GrobidProperties.java
@@ -7,17 +7,19 @@
 import org.grobid.core.engines.tagging.GrobidCRFEngine;
 import org.grobid.core.exceptions.GrobidPropertyException;
 import org.grobid.core.exceptions.GrobidResourceException;
-import org.grobid.core.utilities.Consolidation.GrobidConsolidationService;
 import org.grobid.core.main.GrobidHomeFinder;
+import org.grobid.core.utilities.Consolidation.GrobidConsolidationService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.*;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.util.Enumeration;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * This class loads contains all names of grobid-properties and provide methods
@@ -298,10 +300,10 @@ private void init() {
         try {
             getProps().load(new FileInputStream(getGrobidPropertiesPath()));
         } catch (IOException exp) {
-            throw new GrobidPropertyException("Cannot open file of grobid.properties at location'" + GROBID_PROPERTY_PATH.getAbsolutePath()
+            throw new GrobidPropertyException("Cannot open file of grobid.properties at location '" + GROBID_PROPERTY_PATH.getAbsolutePath()
                 + "'", exp);
         } catch (Exception exp) {
-            throw new GrobidPropertyException("Cannot open file of grobid properties" + getGrobidPropertiesPath().getAbsolutePath(), exp);
+            throw new GrobidPropertyException("Cannot open file of grobid properties " + getGrobidPropertiesPath().getAbsolutePath(), exp);
         }
 
         getProps().putAll(getEnvironmentVariableOverrides(System.getenv()));
@@ -312,6 +314,23 @@ private void init() {
         loadCrfEngine();
     }
 
+    /** Return the distinct values of all the engines that are needed */
+    public static Set<GrobidCRFEngine> getDistinctModels() {
+        final Set<GrobidCRFEngine> modelSpecificEngines = new HashSet<>(getModelSpecificEngines());
+        modelSpecificEngines.add(getGrobidCRFEngine());
+
+        return modelSpecificEngines;
+    }
+
+    /** Return the distinct values of all the engines specified in the individual model configuration in the property file **/
+    public static Set<GrobidCRFEngine> getModelSpecificEngines() {
+        return getProps().keySet().stream()
+            .filter(k -> ((String) k).startsWith(GrobidPropertyKeys.PROP_GROBID_CRF_ENGINE + '.'))
+            .map(k -> GrobidCRFEngine.get(StringUtils.lowerCase(getPropertyValue((String) k))))
+            .distinct()
+            .collect(Collectors.toSet());
+    }
+
     protected static void loadCrfEngine() {
         grobidCRFEngine = GrobidCRFEngine.get(getPropertyValue(GrobidPropertyKeys.PROP_GROBID_CRF_ENGINE,
             GrobidCRFEngine.WAPITI.name()));