Skip to content
Permalink
Browse files
OAK-9756 statistics for the IndexSanityChecker (#563)
* OAK-9756 statistics for the IndexSanityChecker

* minor adjustments, thanks @thomasmueller

* Update oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java

Co-authored-by: Fabrizio Fortino <fabrizio.fortino@gmail.com>

Co-authored-by: Fabrizio Fortino <fabrizio.fortino@gmail.com>
  • Loading branch information
joerghoh and fabriziofortino committed May 10, 2022
1 parent fbdad6b commit c25e78d70615bbb8a6bf677055ba309c841ebd26
Showing 3 changed files with 65 additions and 8 deletions.
@@ -109,6 +109,7 @@ public class IndexCopier implements CopyOnReadStatsMBean, Closeable {
private volatile boolean closed;
private final IndexRootDirectory indexRootDirectory;
private final Set<String> validatedIndexPaths = Sets.newConcurrentHashSet();
private final IndexSanityChecker.IndexSanityStatistics indexSanityStatistics = new IndexSanityChecker.IndexSanityStatistics();

public IndexCopier(Executor executor, File indexRootDir) throws IOException {
this(executor, indexRootDir, false);
@@ -233,7 +234,7 @@ private void checkIntegrity(String indexPath, Directory local, Directory remote)
//Also at this time its required that state in local dir should exactly same as
//one in remote dir
synchronized (validatedIndexPaths){
new IndexSanityChecker(indexPath, local, remote).check();
new IndexSanityChecker(indexPath, local, remote).check(indexSanityStatistics);
validatedIndexPaths.add(indexPath);
}
}
@@ -48,8 +48,9 @@ public IndexSanityChecker(String indexPath, Directory local, Directory remote) {
this.indexPath = indexPath;
}

public boolean check() throws IOException {
public boolean check(IndexSanityStatistics stats) throws IOException {
boolean allFine = true;
long start = System.currentTimeMillis();
//TODO Add support for checksum based checks
if (isThereASizeMismatch()){
//In case of any mismatch just purge all local files
@@ -63,15 +64,18 @@ public boolean check() throws IOException {
}
}
}
stats.addDuration(System.currentTimeMillis() - start);
stats.addIndexSize(localDirSize);

if (allFine) {
log.info("Local index directory content found to be valid for index [{}]. " +
"Stats Local: {} files ({}), Remote: {} files ({})", indexPath,
"Stats Local: {} files ({}), Remote: {} files ({}), accumulated statistics on checking index sanity: {}", indexPath,
localFileCount, IOUtils.humanReadableByteCount(localDirSize),
remoteFileCount, IOUtils.humanReadableByteCount(remoteDirSize));
remoteFileCount, IOUtils.humanReadableByteCount(remoteDirSize),
stats.toString());
} else {
log.warn("Local index directory content were not found to be in sync with remote for index [{}]. " +
"Local directory content has been purged and would be synced again from remote", indexPath);
"Local directory content has been purged and would be synced again from remote.", indexPath);
}
return allFine;
}
@@ -118,4 +122,54 @@ private static void deleteAllFiles(Directory dir) throws IOException {
dir.deleteFile(fileName);
}
}


/**
* Accumulated statistics across multiple invocations of the IndexSanityChecker
*
*/
public static class IndexSanityStatistics {

long totalDurationInMs;
long totalIndexSize;

/**
* Record the time spend to check the sanity of indexes
* @param milis the time in miliseconds
*/
public void addDuration(long milis) {
totalDurationInMs += milis;
}

/**
* Return the accumulated time for checking the sanity of indexes
* @return the total duration in miliseconds
*/
public long getAccumulatedDuration() {
return totalDurationInMs;
}

/**
* Record the size of an index considered
* @param bytes
*/
public void addIndexSize (long bytes) {
totalIndexSize += bytes;
}

/**
* Return the accumulated index size
* @return index size in bytes
*/
public long getAccumulatedIndexSize() {
return totalIndexSize;
}

@Override
public String toString() {
return String.format("[duration: %d ms, "
+ "checked index size: %d bytes (%s)]", totalDurationInMs,
totalIndexSize, IOUtils.humanReadableByteCount(totalIndexSize));
}
}
}
@@ -36,13 +36,15 @@ public class IndexSanityCheckerTest {

private Directory local = new RAMDirectory();
private Directory remote = new RAMDirectory();

private IndexSanityChecker.IndexSanityStatistics stats = new IndexSanityChecker.IndexSanityStatistics();

@Test
public void validDirs() throws Exception{
byte[] t1 = writeFile(local, "t1", 100);
writeFile(remote, "t1", t1);

assertTrue(new IndexSanityChecker("/foo", local, remote).check());
assertTrue(new IndexSanityChecker("/foo", local, remote).check(stats));

assertTrue(local.fileExists("t1"));
assertTrue(remote.fileExists("t1"));
@@ -59,7 +61,7 @@ public void sizeMismatch() throws Exception{
byte[] t3R = writeFile(remote, "t3", 140);
writeFile(local, "t3", t3R);

assertFalse(new IndexSanityChecker("/foo", local, remote).check());
assertFalse(new IndexSanityChecker("/foo", local, remote).check(stats));

assertTrue(remote.fileExists("t3"));

@@ -74,7 +76,7 @@ public void extraLocalFiles() throws Exception{
byte[] t3R = writeFile(remote, "t3", 140);
writeFile(local, "t3", t3R);

new IndexSanityChecker("/foo", local, remote).check();
new IndexSanityChecker("/foo", local, remote).check(stats);

//t1 exist in local but not in remote
//it must be removed

0 comments on commit c25e78d

Please sign in to comment.