Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

OAK-9754 increase default dump threshold to 16 MB and expose as system property #551

Merged
merged 1 commit into from Apr 29, 2022
Merged

OAK-9754 increase default dump threshold to 16 MB and expose as system property #551

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 11 additions & 10 deletions ...a/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
View file
Open in desktop
Expand Up @@ -19,6 +19,14 @@

package org.apache.jackrabbit.oak.index.indexer.document.flatfile;

import com.google.common.collect.Iterables;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.index.indexer.document.CompositeException;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
Expand All @@ -28,14 +36,6 @@
import java.util.Set;
import java.util.function.Predicate;

import com.google.common.collect.Iterables;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.index.indexer.document.CompositeException;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static java.util.Collections.unmodifiableSet;

public class FlatFileNodeStoreBuilder {
Expand Down Expand Up @@ -88,7 +88,8 @@ public class FlatFileNodeStoreBuilder {
*/
static final String OAK_INDEXER_MAX_SORT_MEMORY_IN_GB = "oak.indexer.maxSortMemoryInGB";
static final int OAK_INDEXER_MAX_SORT_MEMORY_IN_GB_DEFAULT = 2;
static final long DEFAULT_DUMP_THRESHOLD = FileUtils.ONE_MB;
static final String OAK_INDEXER_DUMP_THRESHOLD_IN_MB = "oak.indexer.dumpThresholdInMB";
static final int OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT = 16;
private final Logger log = LoggerFactory.getLogger(getClass());
private List<Long> lastModifiedBreakPoints;
private final File workDir;
Expand All @@ -101,7 +102,7 @@ public class FlatFileNodeStoreBuilder {
private long entryCount = 0;
private File flatFileStoreDir;
private final MemoryManager memoryManager;
private long dumpThreshold = DEFAULT_DUMP_THRESHOLD;
private long dumpThreshold = Integer.getInteger(OAK_INDEXER_DUMP_THRESHOLD_IN_MB, OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT) * FileUtils.ONE_MB;
Ewocker marked this conversation as resolved.
Show resolved Hide resolved
private Predicate<String> pathPredicate = path -> true;

private final boolean useZip = Boolean.parseBoolean(System.getProperty(OAK_INDEXER_USE_ZIP, "true"));
Expand Down
8 changes: 4 additions & 4 deletions ...rabbit/oak/index/indexer/document/flatfile/MultithreadedTraverseWithSortStrategyTest.java
View file
Open in desktop
Expand Up @@ -18,8 +18,8 @@
*/
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;

import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.index.indexer.document.LastModifiedRange;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.junit.Test;
Expand All @@ -32,8 +32,8 @@
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;

import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentTraverser.TraversingRange;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.MultithreadedTraverseWithSortStrategy.DirectoryHelper;
import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentTraverser.TraversingRange;
import static org.junit.Assert.assertEquals;

public class MultithreadedTraverseWithSortStrategyTest {
Expand All @@ -44,7 +44,7 @@ public void initialRanges() throws IOException {
List<TraversingRange> ranges = new ArrayList<>();
MultithreadedTraverseWithSortStrategy mtws = new MultithreadedTraverseWithSortStrategy(null,
lastModifiedBreakpoints, null, null, null, null, true, null,
FlatFileNodeStoreBuilder.DEFAULT_DUMP_THRESHOLD, path -> true) {
FlatFileNodeStoreBuilder.OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT * FileUtils.ONE_MB, path -> true) {

@Override
void addTask(TraversingRange range, NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory, BlobStore blobStore, ConcurrentLinkedQueue<String> completedTasks) throws IOException {
Expand Down Expand Up @@ -104,7 +104,7 @@ public void rangesDuringResume() throws IOException {
List<TraversingRange> ranges = new ArrayList<>();
MultithreadedTraverseWithSortStrategy mtws = new MultithreadedTraverseWithSortStrategy(null,
null, null, null, null, workDirs, true, null,
FlatFileNodeStoreBuilder.DEFAULT_DUMP_THRESHOLD, path -> true) {
FlatFileNodeStoreBuilder.OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT * FileUtils.ONE_MB, path -> true) {
@Override
void addTask(TraversingRange range, NodeStateEntryTraverserFactory nodeStateEntryTraverserFactory,
BlobStore blobStore, ConcurrentLinkedQueue<String> completedTasks) throws IOException {
Expand Down
6 changes: 3 additions & 3 deletions ...va/org/apache/jackrabbit/oak/index/indexer/document/flatfile/TraverseAndSortTaskTest.java
View file
Open in desktop
Expand Up @@ -35,11 +35,11 @@
import java.util.List;
import java.util.Queue;
import java.util.concurrent.Callable;
import java.util.concurrent.Phaser;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Phaser;

import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentTraverser.TraversingRange;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.MultithreadedTraverseWithSortStrategy.DirectoryHelper;
import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentTraverser.TraversingRange;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
Expand Down Expand Up @@ -70,7 +70,7 @@ public void taskSplit() throws IOException {
File store = new File("target/" + this.getClass().getSimpleName() + "-" + System.currentTimeMillis());
TraverseAndSortTask tst = new TraverseAndSortTask(traversingRange, null, null, store, true,
new LinkedList<>(Collections.singletonList("1")), newTaskQueue, phaser, new NodeStateEntryTraverserFactoryImpl(), mockMemManager,
FlatFileNodeStoreBuilder.DEFAULT_DUMP_THRESHOLD, new LinkedBlockingQueue<File>(), path -> true);
FlatFileNodeStoreBuilder.OAK_INDEXER_DUMP_THRESHOLD_IN_MB_DEFAULT * FileUtils.ONE_MB, new LinkedBlockingQueue<File>(), path -> true);

NodeStateEntry mockEntry = Mockito.mock(NodeStateEntry.class);
long lastModified = (lmRange.getLastModifiedFrom() + lmRange.getLastModifiedTo())/2;
Expand Down