Permalink
Browse files

version 1.2.6: tuning GenericTypeahead performance

  • Loading branch information...
1 parent 95edac1 commit 0ac041a9c17ad52473fe6a391c8aaeae329a2775 @krati krati committed Sep 20, 2012
View
@@ -23,7 +23,7 @@ groupId: com.sna-projects.cleo
artifactId: cleo
-version: 1.2.5
+version: 1.2.6
### Code Examples:
View
@@ -10,7 +10,7 @@
<groupId>com.sna-projects.cleo</groupId>
<artifactId>cleo</artifactId>
<packaging>bundle</packaging>
- <version>1.2.5</version>
+ <version>1.2.6</version>
<name>cleo</name>
<description>Flexible, partial, out-of-order real-time typeahead search</description>
@@ -148,7 +148,7 @@
<dependency>
<groupId>com.sna-projects.krati</groupId>
<artifactId>krati</artifactId>
- <version>0.4.7.1</version>
+ <version>0.4.7.2</version>
<scope>compile</scope>
</dependency>
@@ -21,6 +21,9 @@
import java.util.Iterator;
import java.util.concurrent.locks.ReentrantLock;
+import org.apache.log4j.Logger;
+
+import krati.PersistableListener;
import krati.core.segment.MemorySegmentFactory;
import krati.core.segment.SegmentFactory;
import krati.io.Serializer;
@@ -33,6 +36,11 @@
*/
public class KratiBufferedInts implements DataStoreInts {
/**
+ * The logger.
+ */
+ private static final Logger logger = Logger.getLogger(KratiBufferedInts.class);
+
+ /**
* The number of integers in buffer.
*/
protected volatile int bufferSize;
@@ -105,6 +113,9 @@ public KratiBufferedInts(int bufferSize, KratiDataStore store) throws Exception
this.extension = initExtension(new MemorySegmentFactory());
this.extInts = new KratiArrayStoreInts(extension);
+ // Initialize buffer persist listener
+ initPersistableListener();
+
// Initialize reentrant locks
lockArray = new ReentrantLock[32];
for(int i = 0; i < lockArray.length; i++) {
@@ -115,29 +126,56 @@ public KratiBufferedInts(int bufferSize, KratiDataStore store) throws Exception
}
/**
+ * Initializes the buffer persist listener.
+ */
+ protected void initPersistableListener() {
+ final PersistableListener originalListener = buffer.getUnderlyingStore().getPersistableListener();
+ buffer.getUnderlyingStore().setPersistableListener(new PersistableListener() {
+ final PersistableListener origin = originalListener;
+ @Override
+ public void beforePersist() {
+ try {
+ extension.persist();
+ if(origin != null) {
+ origin.beforePersist();
+ }
+ } catch(Exception e) {
+ logger.error("failed on calling beforePersist", e);
+ }
+ }
+
+ @Override
+ public void afterPersist() {
+ if(origin != null) {
+ origin.afterPersist();
+ }
+ }
+ });
+ }
+
+ /**
* Initializes the extension store.
*
* @throws Exception
*/
protected KratiArrayStore initExtension(SegmentFactory segmentFactory) throws Exception {
- File dbHome = new File(buffer.getStoreHome(), "store-ext");
-
+ final File extHomeDir = new File(buffer.getStoreHome(), "store-ext");
final int initialCapacity = buffer.getUnderlyingStore().capacity();
- final int batchSize = buffer.getUpdateBatchSize();
+ final int batchSize = 1000;
final int numSyncBatches = 100;
int segmentFileSizeMB = 128;
double segmentCompactFactor = 0.5;
- KratiArrayStore mainStore = new KratiArrayStore(
+ KratiArrayStore extStore = new KratiArrayStore(
initialCapacity,
batchSize,
numSyncBatches,
- dbHome,
+ extHomeDir,
segmentFactory,
segmentFileSizeMB,
segmentCompactFactor);
- return mainStore;
+ return extStore;
}
protected int getExtensionIndex(String key) {
@@ -28,6 +28,7 @@
import org.apache.log4j.Logger;
import krati.Persistable;
+import krati.PersistableListener;
import krati.core.segment.SegmentFactory;
import krati.store.IndexedDataStore;
@@ -45,8 +46,7 @@
private final File storeHomeDir;
private final int updateBatchSize;
private volatile long lwMark = 0;
- private volatile long hwMark = 0;
- private volatile long counter= 0;
+ private volatile long hwMark = 0;
public KratiDataStore(File storeDir,
int initialCapacity,
@@ -84,6 +84,7 @@ public KratiDataStore(File storeDir,
this.waterMarksFile = new File(storeDir, "waterMarks");
this.waterMarksFileOriginal = new File(storeDir, "waterMarks.original");
this.initWaterMarks();
+ this.initPersistableListener();
}
protected int getInitLevel(int initialCapacity) {
@@ -169,13 +170,15 @@ protected void syncWaterMarks() {
}
}
- protected void internalPersist(long scn) throws Exception {
- counter++;
- hwMark = Math.max(hwMark, scn);
- if(counter % updateBatchSize == 0) {
- indexedStore.persist();
- syncWaterMarks();
- }
+ protected void initPersistableListener() {
+ indexedStore.setPersistableListener(new PersistableListener() {
+ @Override
+ public void beforePersist() {}
+
+ @Override
+ public void afterPersist() {
+ syncWaterMarks();
+ }});
}
public String getStatus() {
@@ -208,13 +211,13 @@ public int getLength(byte[] key) {
public synchronized boolean put(byte[] key, byte[] value, long scn) throws Exception {
boolean b = indexedStore.put(key, value);
- internalPersist(scn);
+ hwMark = Math.max(hwMark, scn);
return b;
}
public synchronized boolean delete(byte[] key, long scn) throws Exception {
boolean b = indexedStore.delete(key);
- internalPersist(scn);
+ hwMark = Math.max(hwMark, scn);
return b;
}
@@ -129,7 +129,7 @@ public static ArrayStoreWeights createArrayStoreWeights(File storeHomeDir, int c
int initialCapacity,
int indexSegmentFileSizeMB, SegmentFactory indexSegmentFactory,
int storeSegmentFileSizeMB, SegmentFactory storeSegmentFactory) throws Exception {
- int batchSize = 5000;
+ int batchSize = 10000;
int numSyncBatches = 20;
KratiDataStore kds = new KratiDataStore(storeHomeDir,
initialCapacity, batchSize, numSyncBatches,

0 comments on commit 0ac041a

Please sign in to comment.