Merge pull request #156 from sbesson/search_max_fileset_size

Fileset indexing: address performance issues and add configurability
ome · Mar 9, 2023 · 5ff911e · 5ff911e
2 parents 955f350 + 23a8902
commit 5ff911e
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 8 deletions.
diff --git a/src/main/java/ome/services/fulltext/FullTextBridge.java b/src/main/java/ome/services/fulltext/FullTextBridge.java
@@ -55,10 +55,8 @@
  *
  * @author Josh Moore, josh at glencoesoftware.com
  * @since 3.0-Beta3
- * @see <a href="https://trac.openmicroscopy.org.uk/ome/FileParsers">Parsers</a
- *      href>
- * @see <a href="https://trac.openmicroscopy.org.uk/ome/SearchBridges">Bridges</a
- *      href>
+ * @see <a href="https://omero.readthedocs.io/en/stable/developers/Search/FileParsers.html">Parsers</a>
+ * @see <a href="https://omero.readthedocs.io/en/stable/developers/Modules/Search/Bridges.html">Bridges</a>
  */
 @Deprecated
 public class FullTextBridge extends BridgeHelper {
@@ -67,6 +65,7 @@ public class FullTextBridge extends BridgeHelper {
     final protected OriginalFilesService files;
     final protected Map<String, FileParser> parsers;
     final protected Class<FieldBridge>[] classes;
+    protected int maxFilesetSize;
 
     /**
      * Since this constructor provides the instance with no way of parsing
@@ -100,8 +99,7 @@ public FullTextBridge(OriginalFilesService files,
      *            set of {@link FieldBridge bridge classes} which will be
      *            instantiated via a no-arg constructor.
      * @see <a
-     *      href="https://trac.openmicroscopy.org.uk/ome/SearchBridges">Bridges</a
-     *      href>
+     *      href="https://omero.readthedocs.io/en/stable/developers/Modules/Search/Bridges.html">Bridges</a>
      */
     @SuppressWarnings("unchecked")
     public FullTextBridge(OriginalFilesService files,
@@ -111,6 +109,15 @@ public FullTextBridge(OriginalFilesService files,
         this.classes = bridgeClasses == null ? new Class[] {} : bridgeClasses;
     }
 
+    /**
+     * Sets the cut-off for indexing filesets
+     * @param maxFilesetSize the maximume fileset size
+     */
+    public void setMaxFilesetSize(int maxFilesetSize) {
+        logger().info("Setting maximum fileset size to {}", maxFilesetSize);
+        this.maxFilesetSize = maxFilesetSize;
+    }
+
     /**
      * Default implementation of the
      * {@link #set(String, Object, Document, LuceneOptions)}
@@ -388,24 +395,39 @@ public void set_folders(final String name, final IObject object,
      *
      * - fileset.entry.clientPath
      * - fileset.entry.name
+     * - fileset.templatePrefix
      */
     public void set_fileset(final String name, final IObject object,
                             final Document document, final LuceneOptions opts) {
         if (object instanceof Image) {
             final Image image = (Image) object;
             final Fileset fileset = image.getFileset();
-            if (fileset == null) {
+            if (fileset == null || maxFilesetSize < 1) {
                 return;
             }
+
+            add(document, "fileset.templatePrefix", fileset.getTemplatePrefix(), opts);
+
             final Iterator<FilesetEntry> entryIterator = fileset.iterateUsedFiles();
+            int index = 0;
             while (entryIterator.hasNext()) {
                 final FilesetEntry entry = entryIterator.next();
                 if (entry == null) {
                     continue;
                 }
+
+                // Skip fileset indexing above a cut-off
+                // As the fileset indexing scales with the number of fileset entries for each
+                // image, this operation can quickly lead to performance degradation notable
+                // in domains like high-content screening where each of 1K-10K images in a plate
+                // can be associated with 10-100K files
+                index++;
+                if (index > maxFilesetSize) {
+                    return;
+                }
+
                 add(document, "fileset.entry.clientPath", entry.getClientPath(), opts);
                 add(document, "fileset.entry.name", entry.getOriginalFile().getName(), opts);
-                add(document, "fileset.templatePrefix", fileset.getTemplatePrefix(), opts);
             }
         }
     }

diff --git a/src/main/resources/ome/services/service-ome.api.Search.xml b/src/main/resources/ome/services/service-ome.api.Search.xml
@@ -52,6 +52,7 @@
     <constructor-arg ref="fileParsers"/>
     <constructor-arg ref="/OMERO/Files"/>
     <constructor-arg value="${omero.search.bridges}"/>
+    <property name="maxFilesetSize" value="${omero.search.max_fileset_size}"/>
   </bean>
 
   <!-- Use "*" as a wildcard parser -->

diff --git a/src/main/resources/omero-server.properties b/src/main/resources/omero-server.properties
@@ -197,6 +197,12 @@ ome.model.screen.PlateAcquisition,ome.model.screen.Well
 # modified.
 omero.search.include_actions=INSERT,UPDATE,REINDEX,DELETE
 
+# Maximum number of fileset entries which will be indexed
+# Increasing this cut-off can lead to indexing performance degradation
+# notably in the high-content screening domain where plates typically
+# contain 1K-10K images associated with 10-100K fileset entries each
+# If set to 0, no fileset entry will be indexed
+omero.search.max_fileset_size=10
 
 ##
 ## Old loader: "persistentEventLogLoader"