Skip to content
Permalink
Browse files
add test cases
  • Loading branch information
Ewocker committed Mar 22, 2022
1 parent 635df52 commit 2d3bc303f17e6898d8c6519978d54307326bd15b
Showing 2 changed files with 142 additions and 43 deletions.
@@ -53,7 +53,7 @@ public class FlatFileNodeStoreBuilder {
* Allowed values are the values from enum {@link SortStrategyType}
*/
public static final String OAK_INDEXER_SORT_STRATEGY_TYPE = "oak.indexer.sortStrategyType";
private static final String OAK_INDEXER_SORTED_FILE_PATH = "oak.indexer.sortedFilePath";
public static final String OAK_INDEXER_SORTED_FILE_PATH = "oak.indexer.sortedFilePath";

/**
* Default value for {@link #PROP_THREAD_POOL_SIZE}
@@ -1,7 +1,7 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;

import com.google.common.collect.Lists;
import jnr.ffi.annotations.In;
import org.apache.commons.io.comparator.SizeFileComparator;
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.junit.After;
import org.junit.Before;
@@ -12,7 +12,6 @@
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedList;
@@ -34,12 +33,11 @@ public class MergeRunnerTest {
.create();
private final String newline = System.lineSeparator();
private final List<File> testFiles = Lists.newArrayList();
private String expectedSortedFileStr = "",
actualSortedFileStr = "";
private final int threadPoolSize = 1,
batchMergeSize = 3;
private final PathElementComparator pathComparator = new PathElementComparator();
private final Comparator<NodeStateHolder> comparator = (e1, e2) -> pathComparator.compare(e1.getPathElements(), e2.getPathElements());
private final Comparator fileSizeComparator = new SizeFileComparator();
private final NodeStateEntryWriter entryWriter = new NodeStateEntryWriter(new MemoryBlobStore());

@Before
@@ -50,20 +48,21 @@ public void setup(){
@After
public void after() {
lc.finished();
testFiles.clear();
}

@Test
public void test() throws IOException {
public void test() throws Exception {
lc.starting();

File tmpDir = new File(FileUtils.getTempDirectory(), Long.toString(System.nanoTime())),
mergeDir = new File(tmpDir, "merge"),
sortedFile = new File(tmpDir, "sorted-file.json");
mergeDir = new File(tmpDir, "merge-reverse"),
sortedFile = new File(tmpDir, "sorted-file.json");
List<String> expectedLogOutput = Lists.newArrayList(),
actualLogOutput = Lists.newArrayList();
actualLogOutput = Lists.newArrayList();

int testFileCount = 13;
generateTestFiles(tmpDir, testFileCount);
String expectedSortedFileStr = generateTestFiles(tmpDir, testFileCount);
assertEquals("expected generated test files number does not match", testFileCount, testFiles.size());

BlockingQueue<File> sortedFiles = new LinkedBlockingQueue<>();
@@ -72,86 +71,184 @@ public void test() throws IOException {
Thread merger = new Thread(mergeRunner, "test-merger");
merger.setDaemon(true);

// Adding test files in predefined order
ArrayList<Integer> filenameOrder = new ArrayList<>(Arrays.asList(7, 1, 11, 9, 8, 6, 12, 4, 5, 13, 3, 2, 10));
for (int filename: filenameOrder.subList(0,4)) {
sortedFiles.add(testFiles.get(filename-1));
}
sortedFiles.addAll(testFiles);
merger.start();
for (int filename: filenameOrder.subList(4,13)) {
sortedFiles.add(testFiles.get(filename-1));
}
sortedFiles.add(MergeRunner.MERGE_POISON_PILL);
mergePhaser.awaitAdvance(0);

actualLogOutput = lc.getLogs();
actualSortedFileStr = FileUtils.readFileToString(sortedFile, UTF_8);
System.out.println(String.join(newline, actualLogOutput));
String actualSortedFileStr = FileUtils.readFileToString(sortedFile, UTF_8);
assertEquals("sorted-file content should be expected", expectedSortedFileStr, actualSortedFileStr);

expectedLogOutput.add("created merge task for intermediate-1 with " + (new ArrayList<File>(){
List<String> intermediateMergeLog = new ArrayList<String>(){
{
add("merge complete for intermediate-1");
add("merge complete for intermediate-2");
add("merge complete for intermediate-3");
}
};
assertTrue("merge complete output should exist", actualLogOutput.containsAll(intermediateMergeLog));
actualLogOutput.removeAll(intermediateMergeLog);

expectedLogOutput.add("created merge task for intermediate-1 with " + (new ArrayList<File>(batchMergeSize){
{
add(testFiles.get(0));
add(testFiles.get(5));
add(testFiles.get(6));
add(testFiles.get(1));
add(testFiles.get(2));
}
}));
expectedLogOutput.add("created merge task for intermediate-2 with " + (new ArrayList<File>(){
expectedLogOutput.add("created merge task for intermediate-2 with " + (new ArrayList<File>(batchMergeSize){
{
add(testFiles.get(3));
add(testFiles.get(4));
add(testFiles.get(7));
add(testFiles.get(5));
}
}));
expectedLogOutput.add("created merge task for intermediate-3 with " + (new ArrayList<File>(){
expectedLogOutput.add("created merge task for intermediate-3 with " + (new ArrayList<File>(batchMergeSize){
{
add(testFiles.get(1));
add(testFiles.get(2));
add(testFiles.get(6));
add(testFiles.get(7));
add(testFiles.get(8));
}
}));
expectedLogOutput.add("Waiting for batch sorting tasks completion");
assertEquals("intermediate merge log output should be expected", String.join(newline, expectedLogOutput), String.join(newline, actualLogOutput.subList(0,4)));
expectedLogOutput.add("There are still 4 sorted files not merged yet");
expectedLogOutput.add("running final batch merge task for final-1 with " + (new ArrayList<File>(batchMergeSize){
{
add(new File(mergeDir, "intermediate-1"));
add(testFiles.get(9));
add(testFiles.get(10));
}
}));
expectedLogOutput.add("running final batch merge task for final-2 with " + (new ArrayList<File>(batchMergeSize){
{
add(testFiles.get(11));
add(testFiles.get(12));
add(new File(mergeDir, "intermediate-2"));
}
}));
expectedLogOutput.add("running final batch merge task for " + sortedFile.getName() + " with " + (new ArrayList<File>(batchMergeSize){
{
add(new File(mergeDir, "intermediate-3"));
add(new File(mergeDir, "final-1"));
add(new File(mergeDir, "final-2"));
}
}));
expectedLogOutput.add("Total batch sorted files length is 18");
assertEquals("final merge log output should be expected", String.join(newline, expectedLogOutput), String.join(newline, actualLogOutput));
}

@Test
public void testReverse() throws Exception {
lc.starting();

File tmpDir = new File(FileUtils.getTempDirectory(), Long.toString(System.nanoTime())),
mergeDir = new File(tmpDir, "merge"),
sortedFile = new File(tmpDir, "sorted-file.json");
List<String> expectedLogOutput = Lists.newArrayList(),
actualLogOutput = Lists.newArrayList();

assertTrue("merge complete output should exist", actualLogOutput.containsAll((new ArrayList<String>(){
int testFileCount = 16;
String expectedSortedFileStr = generateTestFiles(tmpDir, testFileCount);
assertEquals("expected generated test files number does not match", testFileCount, testFiles.size());

BlockingQueue<File> sortedFiles = new LinkedBlockingQueue<>();
Phaser mergePhaser = new Phaser(1);
Runnable mergeRunner = new MergeRunner(sortedFile, sortedFiles, mergeDir, comparator, mergePhaser, batchMergeSize, threadPoolSize, false);
Thread merger = new Thread(mergeRunner, "test-merger");
merger.setDaemon(true);

Collections.reverse(testFiles);
sortedFiles.addAll(testFiles);
merger.start();
sortedFiles.add(MergeRunner.MERGE_POISON_PILL);
mergePhaser.awaitAdvance(0);

actualLogOutput = lc.getLogs();
System.out.println(String.join(newline, actualLogOutput));
String actualSortedFileStr = FileUtils.readFileToString(sortedFile, UTF_8);
assertEquals("sorted-file content should be expected", expectedSortedFileStr, actualSortedFileStr);

Collections.reverse(testFiles);
List<String> intermediateMergeLog = new ArrayList<String>(){
{
add("merge complete for intermediate-1");
add("merge complete for intermediate-2");
add("merge complete for intermediate-3");
add("merge complete for intermediate-4");
}
})));
};
assertTrue("merge complete output should exist", actualLogOutput.containsAll(intermediateMergeLog));
actualLogOutput.removeAll(intermediateMergeLog);

expectedLogOutput.clear();
expectedLogOutput.add("There are still 4 sorted files not merged yet");
expectedLogOutput.add("running final batch merge task for final-1 with " + (new ArrayList<File>(){
expectedLogOutput.add("created merge task for intermediate-1 with " + (new ArrayList<File>(batchMergeSize){
{
add(testFiles.get(9));
add(testFiles.get(10));
add(testFiles.get(11));
add(testFiles.get(12));
}
}));
expectedLogOutput.add("running final batch merge task for final-2 with " + (new ArrayList<File>(){
expectedLogOutput.add("created merge task for intermediate-2 with " + (new ArrayList<File>(batchMergeSize){
{
add(testFiles.get(12));
add(new File(mergeDir, "intermediate-1"));
add(testFiles.get(7));
add(testFiles.get(8));
add(testFiles.get(9));
}
}));
expectedLogOutput.add("created merge task for intermediate-3 with " + (new ArrayList<File>(batchMergeSize){
{
add(testFiles.get(4));
add(testFiles.get(5));
add(testFiles.get(6));
}
}));
expectedLogOutput.add("created merge task for intermediate-4 with " + (new ArrayList<File>(batchMergeSize){
{
add(testFiles.get(1));
add(testFiles.get(2));
add(testFiles.get(3));
}
}));
expectedLogOutput.add("Waiting for batch sorting tasks completion");
expectedLogOutput.add("There are still 4 sorted files not merged yet");
expectedLogOutput.add("running final batch merge task for final-1 with " + (new ArrayList<File>(batchMergeSize){
{
add(testFiles.get(0));
add(new File(mergeDir, "intermediate-4"));
add(testFiles.get(13));
}
}));
expectedLogOutput.add("running final batch merge task for final-2 with " + (new ArrayList<File>(batchMergeSize){
{
add(testFiles.get(14));
add(testFiles.get(15));
add(new File(mergeDir, "intermediate-3"));
}
}));
expectedLogOutput.add("running final batch merge task for " + sortedFile.getName() + " with " + (new ArrayList<File>(){
expectedLogOutput.add("running final batch merge task for final-3 with " + (new ArrayList<File>(batchMergeSize){
{
add(new File(mergeDir, "intermediate-2"));
add(new File(mergeDir, "final-1"));
add(new File(mergeDir, "intermediate-2"));
add(new File(mergeDir, "intermediate-1"));
}
}));
expectedLogOutput.add("running final batch merge task for " + sortedFile.getName() + " with " + (new ArrayList<File>(batchMergeSize){
{
add(new File(mergeDir, "final-2"));
add(new File(mergeDir, "final-3"));
}
}));
expectedLogOutput.add("Total batch sorted files length is 18");
assertEquals("final merge log output should be expected", String.join(newline, expectedLogOutput), String.join(newline, actualLogOutput.subList(7, 12)));
expectedLogOutput.add("Total batch sorted files length is 23");
assertEquals("final merge log output should be expected", String.join(newline, expectedLogOutput), String.join(newline, actualLogOutput));
}

// The function will generate <fileCount> files where the filename equals number of lines in the file.
private void generateTestFiles(File tmpDir, int fileCount) throws IOException {
private String generateTestFiles(File tmpDir, int fileCount) throws IOException {
LinkedList<NodeStateHolder> resultNodeState = new LinkedList<>();
List<Integer> input = IntStream.rangeClosed(1, (1+fileCount)*fileCount/2).boxed().collect(Collectors.toList());
List<Integer> input = IntStream.rangeClosed(1, (1 + fileCount) * fileCount / 2).boxed().collect(Collectors.toList());
Collections.shuffle(input);
// cannot have duplicate because externalMerge does not take care of duplicate items
for (int fname = 1; fname <= fileCount; fname++) {
LinkedList<NodeStateHolder> tmpNodeState = new LinkedList<>();
for (int line = 0; line < fname; line++) {
@@ -171,11 +268,13 @@ private void generateTestFiles(File tmpDir, int fileCount) throws IOException {
testFiles.add(testFile);
}

String expectedSortedFileStr = "";
resultNodeState.sort(comparator);
while (!resultNodeState.isEmpty()) {
NodeStateHolder h = resultNodeState.removeFirst();
String text = entryWriter.toString(h.getPathElements(), h.getLine());
expectedSortedFileStr += text + newline;
}
return expectedSortedFileStr;
}
}

0 comments on commit 2d3bc30

Please sign in to comment.