Permalink
Browse files

Adding segment_factory config option to KratiStorage and logging JVM …

…info at IndexEngine startup
  • Loading branch information...
1 parent 67f231e commit 87748e54c910629686478727a6d6b7d9322248d4 @clamprecht committed May 15, 2012
@@ -597,6 +597,13 @@ public static void main(String[] args) throws IOException{
// create the parser
CommandLineParser parser = new PosixParser();
try {
+ // get JVM info
+ Runtime runtime = Runtime.getRuntime();
+ // maxMemory is the -Xmx value
+ String jvmVendor = System.getProperty("java.vm.vendor");
+ logger.info("JVM info: " + jvmVendor+", " + runtime.availableProcessors() + " processors, JVM max memory: " +
+ runtime.maxMemory());
+
// parse the command line arguments
CommandLine line = parser.parse( getOptions(), args );
if (line.hasOption("help")) {
@@ -16,6 +16,7 @@
import krati.core.StoreConfig;
import krati.core.StoreFactory;
//import krati.core.segment.ChannelSegmentFactory;
+import krati.core.segment.ChannelSegmentFactory;
import krati.core.segment.SegmentFactory;
import krati.core.segment.WriteBufferSegmentFactory;
import krati.store.DataStore;
@@ -29,13 +30,13 @@
private final Charset UTF8_CHARSET;
private final DataStore<byte[], byte[]> store;
- public KratiStorage(File cacheDirectory, int initialCapacity, int segmentFileSizeMB) throws Exception {
+ KratiStorage(File cacheDirectory, String segmentFactoryString, int initialCapacity, int segmentFileSizeMB) throws Exception {
Preconditions.checkArgument(initialCapacity >= 50000, "Initial capacity must be at least 50000");
Preconditions.checkArgument(segmentFileSizeMB >= 16, "segmentFileSizeMB must be at least 16");
Preconditions.checkArgument(segmentFileSizeMB <= 512, "segmentFileSizeMB must be <= 512");
logger.info("Creating a Krati store with initialCapacity " + initialCapacity +
- " and segmentFileSizeMB " + segmentFileSizeMB);
+ " and segmentFileSizeMB " + segmentFileSizeMB + ", segmentFactory: " + segmentFactoryString);
if (!cacheDirectory.exists()) {
logger.info("Creating new storage directory: " + cacheDirectory.getAbsolutePath());
cacheDirectory.mkdirs();
@@ -50,10 +51,15 @@ public KratiStorage(File cacheDirectory, int initialCapacity, int segmentFileSiz
StoreConfig config = new StoreConfig(cacheDirectory, initialCapacity);
config.setBatchSize(10000); // 10000 performed better loading index
config.setNumSyncBatches(5);
- // ChannelSegment uses less RAM, with possibly lower write performance
- //SegmentFactory segmentFactory = new ChannelSegmentFactory();
- // WriteBufferSegment may give better write performance, but uses more memory
- SegmentFactory segmentFactory = new WriteBufferSegmentFactory(segmentFileSizeMB);
+ final SegmentFactory segmentFactory;
+ if ("channel".equals(segmentFactoryString)) {
+ // ChannelSegment uses less RAM, with possibly lower write performance
+ segmentFactory = new ChannelSegmentFactory();
+ } else { // default
+ // WriteBufferSegment may give better write performance, but uses more memory
+ // The constructor immediately allocates three ByteBuffers based on segmentFileSizeMB
+ segmentFactory = new WriteBufferSegmentFactory(segmentFileSizeMB);
+ }
config.setSegmentFactory(segmentFactory);
config.setSegmentFileSizeMB(segmentFileSizeMB);
this.store = StoreFactory.createIndexedDataStore(config);
@@ -103,6 +109,7 @@ public void close() throws IOException {
public static final String DIR = "dir";
public static final String INITIAL_CAPACITY = "initial_capacity";
public static final String SEGMENT_FILESIZE_MB = "segment_filesize";
+ public static final String SEGMENT_FACTORY = "segment_factory";
@Override
public DocumentStorage fromConfiguration(Map<?, ?> config) {
@@ -117,16 +124,20 @@ public DocumentStorage fromConfiguration(Map<?, ?> config) {
// defaults
int initialCapacity = 1000000;
int segmentFileSizeMB = 64;
+ String segmentFactory = "writebuffer";
if (config.containsKey(INITIAL_CAPACITY)) {
initialCapacity = Integer.valueOf(config.get(INITIAL_CAPACITY).toString());
}
if (config.containsKey(SEGMENT_FILESIZE_MB)) {
segmentFileSizeMB = Integer.valueOf(config.get(SEGMENT_FILESIZE_MB).toString());
}
+ if (config.containsKey(SEGMENT_FACTORY)) {
+ segmentFactory = config.get(SEGMENT_FACTORY).toString();
+ }
try {
- return new KratiStorage(storageDir, initialCapacity, segmentFileSizeMB);
+ return new KratiStorage(storageDir, segmentFactory, initialCapacity, segmentFileSizeMB);
} catch (Exception e) {
throw new RuntimeException("while creating a KratiStorage: " + e.getMessage(), e);
}

0 comments on commit 87748e5

Please sign in to comment.