Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Make max file size for Indexer a config setting

  • Loading branch information...
commit 7279ec6423c32c2a10dd88ad6597a8ca38a4a8c2 1 parent 12825e1
@jballanc jballanc authored jballanc committed
View
8 components/server/resources/ome/services/service-ome.api.Search.xml
@@ -66,8 +66,12 @@
</constructor-arg>
</bean>
- <bean id="fileParser" class="ome.services.fulltext.FileParser"/>
- <bean id="pdfParser" class="ome.services.fulltext.PdfParser"/>
+ <bean id="fileParser" class="ome.services.fulltext.FileParser">
+ <property name="maxFileSize" value="${omero.search.max_file_size}"/>
+ </bean>
+ <bean id="pdfParser" class="ome.services.fulltext.PdfParser">
+ <property name="maxFileSize" value="${omero.search.max_file_size}"/>
+ </bean>
<alias name="persistentEventLogLoader" alias="eventLogLoader"/>
View
18 components/server/src/ome/services/fulltext/FileParser.java
@@ -13,6 +13,7 @@
import java.io.Reader;
import java.util.Iterator;
import java.util.NoSuchElementException;
+import java.lang.Runtime;
import ome.services.messages.RegisterServiceCleanupMessage;
import ome.system.OmeroContext;
@@ -39,11 +40,23 @@
protected OmeroContext context;
+ protected long maxFileSize;
+
public void setApplicationContext(ApplicationContext arg0)
throws BeansException {
context = (OmeroContext) arg0;
}
+ public void setMaxFileSize(Long size) {
+ if (size.floatValue() / Runtime.getRuntime().maxMemory() > 0.5) {
+ log.warn("Indexer maximum file size is set to more than half of "
+ + "total heap size. Excessively large text files may "
+ + "cause search index corruption. Consider decreasing the "
+ + "maximum file size or increasing the Indexer heap size.");
+ }
+ this.maxFileSize = size;
+ }
+
/**
* {@link Iterable} which returns an empty {@link Iterator}. This will be
* used in case
@@ -96,8 +109,9 @@ public void remove() {
return EMPTY;
}
- if (file.length() > (50 * 1024 * 1024)) {
- log.warn("File too large for indexing. Skipping: " + file.getAbsoluteFile());
+ if (file.length() > this.maxFileSize) {
+ log.warn("File too large for indexing. Skipping: "
+ + file.getAbsoluteFile());
return EMPTY;
}
View
5 etc/omero.properties
@@ -164,6 +164,11 @@ omero.search.analyzer=ome.services.fulltext.FullTextAnalyzer
# Maximum number of OR-clauses to which a single search can expand
omero.search.maxclause=4096
+# Maximum file size for text indexing
+# This should not be set to more than half of the Indexer heap space.
+# By default, 125 MB.
+omero.search.max_file_size=131072000
+
# Indexing takes place on all EventLogs as they occur in the database.
# The types listed here will be skipped if they appear in the "entityType"
# field of the EventLog table.
Please sign in to comment.
Something went wrong with that request. Please try again.