Skip to content
Permalink
Browse files
Merge pull request #551 from Ewocker/OAK-9754
OAK-9754 increase default dump threshold to 16 MB and expose as system property
  • Loading branch information
thomasmueller committed Apr 29, 2022
2 parents 2cc36a1 + 926a774 commit bcab8d29511775c31218f780735939f7faf1c6b7
Showing 3 changed files with 18 additions and 17 deletions.
@@ -19,6 +19,14 @@

package org.apache.jackrabbit.oak.index.indexer.document.flatfile;

import com.google.common.collect.Iterables;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.index.indexer.document.CompositeException;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@@ -28,14 +36,6 @@
import java.util.Set;
import java.util.function.Predicate;

import com.google.common.collect.Iterables;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.index.indexer.document.CompositeException;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static java.util.Collections.unmodifiableSet;

public class FlatFileNodeStoreBuilder {
@@ -88,7 +88,8 @@ public class FlatFileNodeStoreBuilder {
*/
static final String OAK_INDEXER_MAX_SORT_MEMORY_IN_GB = "oak.indexer.maxSortMemoryInGB";
static final int OAK_INDEXER_MAX_SORT_MEMORY_IN_GB_DEFAULT = 2;
static final long DEFAULT_DUMP_THRESHOLD = FileUtils.ONE_MB;
static final String OAK_INDEXER_DUMP_THRESHOLD_IN_MB = "oak.indexer.dumpThresholdInMB";
static final int OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT = 16;
private final Logger log = LoggerFactory.getLogger(getClass());
private List<Long> lastModifiedBreakPoints;
private final File workDir;
@@ -101,7 +102,7 @@ public class FlatFileNodeStoreBuilder {
private long entryCount = 0;
private File flatFileStoreDir;
private final MemoryManager memoryManager;
private long dumpThreshold = DEFAULT_DUMP_THRESHOLD;
private long dumpThreshold = Integer.getInteger(OAK_INDEXER_DUMP_THRESHOLD_IN_MB, OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT) * FileUtils.ONE_MB;
private Predicate<String> pathPredicate = path -> true;

private final boolean useZip = Boolean.parseBoolean(System.getProperty(OAK_INDEXER_USE_ZIP, "true"));
@@ -18,8 +18,8 @@
*/
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;

import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.index.indexer.document.LastModifiedRange;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.junit.Test;
@@ -32,8 +32,8 @@
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;

import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentTraverser.TraversingRange;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.MultithreadedTraverseWithSortStrategy.DirectoryHelper;
import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentTraverser.TraversingRange;
import static org.junit.Assert.assertEquals;

public class MultithreadedTraverseWithSortStrategyTest {
@@ -44,7 +44,7 @@ public void initialRanges() throws IOException {
List<TraversingRange> ranges = new ArrayList<>();
MultithreadedTraverseWithSortStrategy mtws = new MultithreadedTraverseWithSortStrategy(null,
lastModifiedBreakpoints, null, null, null, null, true, null,
FlatFileNodeStoreBuilder.DEFAULT_DUMP_THRESHOLD, path -> true) {
FlatFileNodeStoreBuilder.OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT * FileUtils.ONE_MB, path -> true) {

@Override
void addTask(TraversingRange range, NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory, BlobStore blobStore, ConcurrentLinkedQueue<String> completedTasks) throws IOException {
@@ -104,7 +104,7 @@ public void rangesDuringResume() throws IOException {
List<TraversingRange> ranges = new ArrayList<>();
MultithreadedTraverseWithSortStrategy mtws = new MultithreadedTraverseWithSortStrategy(null,
null, null, null, null, workDirs, true, null,
FlatFileNodeStoreBuilder.DEFAULT_DUMP_THRESHOLD, path -> true) {
FlatFileNodeStoreBuilder.OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT * FileUtils.ONE_MB, path -> true) {
@Override
void addTask(TraversingRange range, NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory,
BlobStore blobStore, ConcurrentLinkedQueue<String> completedTasks) throws IOException {
@@ -35,11 +35,11 @@
import java.util.List;
import java.util.Queue;
import java.util.concurrent.Callable;
import java.util.concurrent.Phaser;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Phaser;

import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentTraverser.TraversingRange;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.MultithreadedTraverseWithSortStrategy.DirectoryHelper;
import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentTraverser.TraversingRange;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
@@ -70,7 +70,7 @@ public void taskSplit() throws IOException {
File store = new File("target/" + this.getClass().getSimpleName() + "-" + System.currentTimeMillis());
TraverseAndSortTask tst = new TraverseAndSortTask(traversingRange, null, null, store, true,
new LinkedList<>(Collections.singletonList("1")), newTaskQueue, phaser, new NodeStateEntryTraverserFactoryImpl(), mockMemManager,
FlatFileNodeStoreBuilder.DEFAULT_DUMP_THRESHOLD, new LinkedBlockingQueue<File>(), path -> true);
FlatFileNodeStoreBuilder.OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT * FileUtils.ONE_MB, new LinkedBlockingQueue<File>(), path -> true);

NodeStateEntry mockEntry = Mockito.mock(NodeStateEntry.class);
long lastModified = (lmRange.getLastModifiedFrom() + lmRange.getLastModifiedTo())/2;

0 comments on commit bcab8d2

Please sign in to comment.