Skip to content

Commit

Permalink
Merge branch 'master' of github.com:senseidb/bobo
Browse files Browse the repository at this point in the history
  • Loading branch information
@javasoze
javasoze committed Feb 1, 2012
2 parents e76f432 + 54ba368 commit c2b87d9
Show file tree
Hide file tree
Showing 6 changed files with 174 additions and 38 deletions.
13 changes: 13 additions & 0 deletions bobo-contrib/src/main/java/com/browseengine/bobo/geosearch/bo/GeoSearchConfig.java
View file
Expand Up @@ -30,6 +30,7 @@ public class GeoSearchConfig {

public static final int DEFAULT_ID_BYTE_COUNT = 16;
private int bytesForId = DEFAULT_ID_BYTE_COUNT;
private int maxIndexSize = Integer.MAX_VALUE;

/**
* Sets the extension for geo indices.
Expand Down Expand Up @@ -108,4 +109,16 @@ public int getBytesForId() {
return bytesForId;
}

/**
* The maximum size the index should be allowed to grow to. Any attempts to
* flush an index larger than this will throw errors. Applies only to the GeoOnlyIndex.
* @param maxIndexSize
*/
public void setMaxIndexSize(int maxIndexSize) {
this.maxIndexSize = maxIndexSize;
}

public int getMaxIndexSize() {
return maxIndexSize;
}
}
11 changes: 11 additions & 0 deletions ...contrib/src/main/java/com/browseengine/bobo/geosearch/solo/bo/IndexTooLargeException.java
View file
@@ -0,0 +1,11 @@
package com.browseengine.bobo.geosearch.solo.bo;

public class IndexTooLargeException extends Exception {
private static final long serialVersionUID = 1L;

public IndexTooLargeException(String indexName, int indexSize, int maxSize) {
super("This index is larger than the maximum allowed index size and will not be flushed to disk" +
". IndexName:" + indexName + "; Actual index size: " + indexSize
+ "; Max index size: " + maxSize);
}
}
9 changes: 8 additions & 1 deletion ...contrib/src/main/java/com/browseengine/bobo/geosearch/solo/index/impl/GeoOnlyIndexer.java
View file
Expand Up @@ -23,6 +23,7 @@
import com.browseengine.bobo.geosearch.index.impl.GeoSegmentReader;
import com.browseengine.bobo.geosearch.index.impl.GeoSegmentWriter;
import com.browseengine.bobo.geosearch.solo.bo.IDGeoRecord;
import com.browseengine.bobo.geosearch.solo.bo.IndexTooLargeException;
import com.browseengine.bobo.geosearch.solo.impl.IDGeoRecordComparator;
import com.browseengine.bobo.geosearch.solo.impl.IDGeoRecordSerializer;

Expand Down Expand Up @@ -88,14 +89,20 @@ public void delete(byte[] uuid) {
* Flushes any requested index updates to directory
*
* @throws IOException
* @throws IndexTooLargeException If the size of the index is larger than the maximum size
*/
public void flush() throws IOException {
public void flush() throws IOException, IndexTooLargeException {
loadCurrentIndex();

for (IDGeoRecord newRecord: newRecords) {
inMemoryIndex.add(newRecord);
}

if (inMemoryIndex.size() > config.getMaxIndexSize()) {
throw new IndexTooLargeException(indexName, inMemoryIndex.size(),
config.getMaxIndexSize());
}

flushInMemoryIndex();
}

Expand Down
2 changes: 1 addition & 1 deletion ...ntrib/src/main/java/com/browseengine/bobo/geosearch/solo/search/impl/GeoOnlySearcher.java
View file
Expand Up @@ -135,7 +135,7 @@ private int calculateMaximumCoordinate(int originalPoint, int delta) {
originalPoint < Integer.MAX_VALUE - delta) {
return originalPoint + delta;
} else {
return Integer.MIN_VALUE;
return Integer.MAX_VALUE;
}
}
}
75 changes: 58 additions & 17 deletions ...rib/src/test/java/com/browseengine/bobo/geosearch/solo/index/impl/GeoOnlyIndexerTest.java
View file
Expand Up @@ -31,6 +31,7 @@
import com.browseengine.bobo.geosearch.index.impl.GeoSegmentReader;
import com.browseengine.bobo.geosearch.index.impl.GeoSegmentWriter;
import com.browseengine.bobo.geosearch.solo.bo.IDGeoRecord;
import com.browseengine.bobo.geosearch.solo.bo.IndexTooLargeException;

/**
*
Expand Down Expand Up @@ -115,14 +116,14 @@ private void expectLockObtain() throws IOException {
}

@Test
public void testIndex_and_flush_emptyIndex() throws IOException {
public void testIndex_and_flush_emptyIndex() throws IOException, IndexTooLargeException {
int countEntries = 50;

index_and_flush(countEntries, false, Collections.<IDGeoRecord>emptySet());
}

@Test
public void testIndex_and_flush_existingIndex() throws IOException {
public void testIndex_and_flush_existingIndex() throws IOException, IndexTooLargeException {
int countEntries = 50;
int idByteLength = GeoSearchConfig.DEFAULT_ID_BYTE_COUNT;

Expand All @@ -141,13 +142,29 @@ public void testIndex_and_flush_existingIndex() throws IOException {
index_and_flush(countEntries, true, existingData);
}

@Test
public void testIndex_and_flush_maxEntries() throws IOException, IndexTooLargeException {
int maxEntries = 100;
config.setMaxIndexSize(maxEntries);

index_and_flush(maxEntries, false, Collections.<IDGeoRecord>emptySet());
}

@Test(expected = IndexTooLargeException.class)
public void testIndex_and_flush_tooManyEntries() throws IOException, IndexTooLargeException {
int maxEntries = 100;
config.setMaxIndexSize(maxEntries);

index_and_flush(maxEntries + 1, false, Collections.<IDGeoRecord>emptySet());
}

private void createIndexFileDummyData() throws IOException {
IndexOutput output = directory.createOutput(indexName + ".geo");
output.writeString("dummyData");
output.flush();
}

private void index_and_flush(int countEntries, final boolean fileExists, final Set<IDGeoRecord> existingData) throws IOException {
private void index_and_flush(final int countEntries, final boolean fileExists, final Set<IDGeoRecord> existingData) throws IOException, IndexTooLargeException {
int idByteLength = GeoSearchConfig.DEFAULT_ID_BYTE_COUNT;
String fieldName = "field1";

Expand All @@ -168,23 +185,27 @@ private void index_and_flush(int countEntries, final boolean fileExists, final S
one(mockGeoSegmentReader).close();
}

one(mockGeoSegmentWriter).close();
if ((countEntries + existingData.size()) <= config.getMaxIndexSize()) {
one(mockGeoSegmentWriter).close();
}

one(mockLock).release();
}
});

indexer.flush();

indexer.close();
try {
indexer.flush();
} finally {
indexer.close();
}

context.assertIsSatisfied();

assertEquals(countEntries + existingData.size(), lastDataFlushed.size());
}

@Test
public void test_delete_and_flush_emptyIndex() throws IOException {
public void test_delete_and_flush_emptyIndex() throws IOException, IndexTooLargeException {
int countEntries = 10;

Set<byte[]> toDelete = new HashSet<byte[]>();
Expand All @@ -196,7 +217,7 @@ public void test_delete_and_flush_emptyIndex() throws IOException {
}

@Test
public void test_delete_and_flush_existingIndex() throws IOException {
public void test_delete_and_flush_existingIndex() throws IOException, IndexTooLargeException {
int countToDelete = 10;

createIndexFileDummyData();
Expand All @@ -219,7 +240,7 @@ public void test_delete_and_flush_existingIndex() throws IOException {
}

@Test
public void test_delete_and_flush_multipleRecords() throws IOException {
public void test_delete_and_flush_multipleRecords() throws IOException, IndexTooLargeException {
int countToDelete = 5;

createIndexFileDummyData();
Expand All @@ -245,7 +266,7 @@ public void test_delete_and_flush_multipleRecords() throws IOException {

private void delete_and_flush(Set<byte[]> toDelete, final boolean fileExists,
final Set<IDGeoRecord> existingData,
int expectedIndexSize) throws IOException {
int expectedIndexSize) throws IOException, IndexTooLargeException {
for (byte[] uuid: toDelete) {
indexer.delete(Arrays.copyOf(uuid, uuid.length));
}
Expand Down Expand Up @@ -275,7 +296,23 @@ private void delete_and_flush(Set<byte[]> toDelete, final boolean fileExists,
}

@Test
public void test_index_delete_and_flush() throws IOException {
public void test_index_delete_and_flush_maxEntries() throws IOException, IndexTooLargeException {
config.setMaxIndexSize(50);
doTest_index_delete_and_flush();
}

@Test(expected = IndexTooLargeException.class)
public void test_index_delete_and_flush_tooManyEntries() throws IOException, IndexTooLargeException {
config.setMaxIndexSize(49);
doTest_index_delete_and_flush();
}

@Test
public void test_index_delete_and_flush() throws IOException, IndexTooLargeException {
doTest_index_delete_and_flush();
}

private void doTest_index_delete_and_flush() throws IOException, IndexTooLargeException {
final Set<IDGeoRecord> existingData = new HashSet<IDGeoRecord>();
Set<byte[]> toDelete = new HashSet<byte[]>();
Set<byte[]> toIndex = new HashSet<byte[]>();
Expand Down Expand Up @@ -315,23 +352,27 @@ public void test_index_delete_and_flush() throws IOException {

one(mockGeoSegmentReader).close();

one(mockGeoSegmentWriter).close();
if (50 <= config.getMaxIndexSize()) {
one(mockGeoSegmentWriter).close();
}

one(mockLock).release();
}
});

indexer.flush();

indexer.close();
try {
indexer.flush();
} finally {
indexer.close();
}

context.assertIsSatisfied();

assertEquals(50, lastDataFlushed.size());
}

@Test(expected = IOException.class)
public void index_and_flush_error_on_read() throws IOException {
public void index_and_flush_error_on_read() throws IOException, IndexTooLargeException {
String fieldName = "field1";

for (int i = 0; i < 10; i++) {
Expand Down

0 comments on commit c2b87d9

Please sign in to comment.