Skip to content
Permalink
Browse files
OAK-9771: IndexWriter config should be customisable from oak (#572)
* OAK-9771: IndexWriter config should be customisable from oak
  • Loading branch information
tihom88 committed May 20, 2022
1 parent 26cf3f2 commit c5eb75abce6cac9fd9e9244d0fd34b1345a285e8
Showing 4 changed files with 86 additions and 0 deletions.
@@ -444,6 +444,12 @@
<artifactId>tomcat-jdbc</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.stefanbirkner</groupId>
<artifactId>system-rules</artifactId>
<version>1.19.0</version>
<scope>test</scope>
</dependency>

</dependencies>
</project>
@@ -75,6 +75,8 @@ public static IndexWriterConfig getIndexWriterConfig(LuceneIndexDefinition defin
config.setCodec(definition.getCodec());
}
config.setRAMBufferSizeMB(writerConfig.getRamBufferSizeMB());
config.setMaxBufferedDeleteTerms(writerConfig.getMaxBufferedDeleteTerms());
config.setRAMPerThreadHardLimitMB(writerConfig.getRamPerThreadHardLimitMB());
return config;
} finally {
thread.setContextClassLoader(loader);
@@ -22,7 +22,23 @@
import org.apache.lucene.index.IndexWriterConfig;

public class LuceneIndexWriterConfig {
/**
* This property will be used to set Lucene's IndexWriter.maxBufferedDeleteTerms
* IndexWriter.maxBufferedDeleteTerms is used to flush buffered data to lucene index.
*/
public final static String MAX_BUFFERED_DELETE_TERMS_KEY = "oak.index.lucene.maxBufferedDeleteTerms";

/**
* This property will be used to set Lucene's IndexWriter.perThreadHardLimitMB.
* IndexWriter.perThreadHardLimitMB is used to flush buffered data to lucene index.
*/
public final static String RAM_PER_THREAD_HARD_LIMIT_MB_KEY = "oak.index.lucene.ramPerThreadHardLimitMB";

private final double ramBufferSizeMB;
private final int maxBufferedDeleteTerms = Integer.getInteger(MAX_BUFFERED_DELETE_TERMS_KEY,
IndexWriterConfig.DISABLE_AUTO_FLUSH);
private final int ramPerThreadHardLimitMB = Integer.getInteger(RAM_PER_THREAD_HARD_LIMIT_MB_KEY,
IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB);

public LuceneIndexWriterConfig() {
this(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
@@ -35,4 +51,12 @@ public LuceneIndexWriterConfig(double ramBufferSizeMB) {
public double getRamBufferSizeMB() {
return ramBufferSizeMB;
}

public int getMaxBufferedDeleteTerms() {
return maxBufferedDeleteTerms;
}

public int getRamPerThreadHardLimitMB() {
return ramPerThreadHardLimitMB;
}
}
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.jackrabbit.oak.plugins.index.lucene;

import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.lucene.index.IndexWriterConfig;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.ProvideSystemProperty;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;

import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT;
import static org.apache.jackrabbit.oak.plugins.index.lucene.writer.IndexWriterUtils.getIndexWriterConfig;
import static org.junit.Assert.assertEquals;

public class LuceneIndexConfigTest {

private NodeState root;

@Rule
public final ProvideSystemProperty updateSystemProperties
= new ProvideSystemProperty("oak.index.lucene.maxBufferedDeleteTerms", "1000")
.and("oak.index.lucene.ramPerThreadHardLimitMB", "100");

@Rule
public final RestoreSystemProperties restoreSystemProperties
= new RestoreSystemProperties();

@Test
public void testIndexWriterConfig() {
root = INITIAL_CONTENT;
NodeBuilder idx = root.builder();
LuceneIndexDefinition definition = new LuceneIndexDefinition(root, idx.getNodeState(), "/foo");
IndexWriterConfig config = getIndexWriterConfig(definition, true);
assertEquals(config.getRAMBufferSizeMB(), 16.0, .01);
assertEquals(100, config.getRAMPerThreadHardLimitMB());
assertEquals(1000, config.getMaxBufferedDeleteTerms());
}
}

0 comments on commit c5eb75a

Please sign in to comment.