Skip to content

Commit

Permalink
Rename from insight to bloom index
Browse files Browse the repository at this point in the history
  • Loading branch information
ragadeeshu committed Sep 12, 2017
1 parent 4e8ba7f commit 074da37
Show file tree
Hide file tree
Showing 11 changed files with 95 additions and 98 deletions.
Expand Up @@ -17,7 +17,7 @@
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
package org.neo4j.kernel.api.impl.insight; package org.neo4j.kernel.api.impl.bloom;


import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField; import org.apache.lucene.document.TextField;
Expand All @@ -30,7 +30,7 @@
* Enumeration representing all possible property types with corresponding encodings and query structures for Lucene * Enumeration representing all possible property types with corresponding encodings and query structures for Lucene
* schema indexes. * schema indexes.
*/ */
enum InsightFieldEncoding enum BloomFieldEncoding
{ {
Number Number
{ {
Expand Down Expand Up @@ -169,7 +169,7 @@ Field encodeField( String name, Value value )
// } // }
}; };


private static final InsightFieldEncoding[] AllEncodings = values(); private static final BloomFieldEncoding[] AllEncodings = values();


abstract String key(); abstract String key();


Expand All @@ -178,9 +178,9 @@ Field encodeField( String name, Value value )
abstract Field encodeField( String name, Value value ); abstract Field encodeField( String name, Value value );




public static InsightFieldEncoding forValue( Value value ) public static BloomFieldEncoding forValue( Value value )
{ {
for ( InsightFieldEncoding encoding : AllEncodings ) for ( BloomFieldEncoding encoding : AllEncodings )
{ {
if ( encoding.canEncode( value ) ) if ( encoding.canEncode( value ) )
{ {
Expand Down
Expand Up @@ -17,7 +17,7 @@
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
package org.neo4j.kernel.api.impl.insight; package org.neo4j.kernel.api.impl.bloom;


import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig;
Expand All @@ -34,13 +34,13 @@


import static org.neo4j.kernel.api.impl.index.LuceneKernelExtensions.directoryFactory; import static org.neo4j.kernel.api.impl.index.LuceneKernelExtensions.directoryFactory;


public class InsightIndex implements AutoCloseable public class BloomIndex implements AutoCloseable
{ {
private final InsightLuceneIndex nodeIndex; private final BloomLuceneIndex nodeIndex;
private final InsightLuceneIndex relationshipIndex; private final BloomLuceneIndex relationshipIndex;
private final String[] properties; private final String[] properties;


public InsightIndex( FileSystemAbstraction fileSystem, File file, String... properties ) throws IOException public BloomIndex( FileSystemAbstraction fileSystem, File file, String... properties ) throws IOException
{ {
this.properties = properties; this.properties = properties;
Factory<IndexWriterConfig> population = () -> IndexWriterConfigs.population( new EnglishAnalyzer() ); Factory<IndexWriterConfig> population = () -> IndexWriterConfigs.population( new EnglishAnalyzer() );
Expand All @@ -50,30 +50,30 @@ public InsightIndex( FileSystemAbstraction fileSystem, File file, String... prop
storageBuilder.withFileSystem( fileSystem ).withIndexIdentifier( "insightNodes" ) storageBuilder.withFileSystem( fileSystem ).withIndexIdentifier( "insightNodes" )
.withDirectoryFactory( directoryFactory( false, fileSystem ) ) .withDirectoryFactory( directoryFactory( false, fileSystem ) )
.withIndexRootFolder( Paths.get( file.getAbsolutePath(),"insightindex" ).toFile() ); .withIndexRootFolder( Paths.get( file.getAbsolutePath(),"insightindex" ).toFile() );
nodeIndex = new InsightLuceneIndex( storageBuilder.build(), partitionFactory, this.properties ); nodeIndex = new BloomLuceneIndex( storageBuilder.build(), partitionFactory, this.properties );
nodeIndex.open(); nodeIndex.open();


storageBuilder = LuceneIndexStorageBuilder.create(); storageBuilder = LuceneIndexStorageBuilder.create();
storageBuilder.withFileSystem( fileSystem ).withIndexIdentifier( "insightRelationships" ) storageBuilder.withFileSystem( fileSystem ).withIndexIdentifier( "insightRelationships" )
.withDirectoryFactory( directoryFactory( false, fileSystem ) ) .withDirectoryFactory( directoryFactory( false, fileSystem ) )
.withIndexRootFolder( Paths.get( file.getAbsolutePath(),"insightindex" ).toFile() ); .withIndexRootFolder( Paths.get( file.getAbsolutePath(),"insightindex" ).toFile() );
relationshipIndex = new InsightLuceneIndex( storageBuilder.build(), partitionFactory, properties ); relationshipIndex = new BloomLuceneIndex( storageBuilder.build(), partitionFactory, properties );
relationshipIndex.open(); relationshipIndex.open();
} }


public InsightIndexTransactionEventUpdater getUpdater() throws IOException public BloomIndexTransactionEventUpdater getUpdater() throws IOException
{ {
WritableDatabaseInsightIndex writableNodeIndex = new WritableDatabaseInsightIndex( nodeIndex ); WritableDatabaseBloomIndex writableNodeIndex = new WritableDatabaseBloomIndex( nodeIndex );
WritableDatabaseInsightIndex writableRelationshipIndex = new WritableDatabaseInsightIndex( relationshipIndex ); WritableDatabaseBloomIndex writableRelationshipIndex = new WritableDatabaseBloomIndex( relationshipIndex );
return new InsightIndexTransactionEventUpdater( writableNodeIndex, writableRelationshipIndex, properties ); return new BloomIndexTransactionEventUpdater( writableNodeIndex, writableRelationshipIndex, properties );
} }


public InsightIndexReader getNodeReader() throws IOException public BloomIndexReader getNodeReader() throws IOException
{ {
return nodeIndex.getIndexReader(); return nodeIndex.getIndexReader();
} }


public InsightIndexReader getRelationshipReader() throws IOException public BloomIndexReader getRelationshipReader() throws IOException
{ {
return relationshipIndex.getIndexReader(); return relationshipIndex.getIndexReader();
} }
Expand Down
Expand Up @@ -17,11 +17,11 @@
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
package org.neo4j.kernel.api.impl.insight; package org.neo4j.kernel.api.impl.bloom;


import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.collection.primitive.PrimitiveLongIterator;


public interface InsightIndexReader extends AutoCloseable public interface BloomIndexReader extends AutoCloseable
{ {
PrimitiveLongIterator query( String... query ); PrimitiveLongIterator query( String... query );
} }
Expand Up @@ -17,7 +17,7 @@
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
package org.neo4j.kernel.api.impl.insight; package org.neo4j.kernel.api.impl.bloom;


import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;


Expand All @@ -31,13 +31,13 @@
import org.neo4j.graphdb.event.TransactionData; import org.neo4j.graphdb.event.TransactionData;
import org.neo4j.graphdb.event.TransactionEventHandler; import org.neo4j.graphdb.event.TransactionEventHandler;


public class InsightIndexTransactionEventUpdater implements TransactionEventHandler<Object> public class BloomIndexTransactionEventUpdater implements TransactionEventHandler<Object>
{ {
private final WritableDatabaseInsightIndex nodeIndex; private final WritableDatabaseBloomIndex nodeIndex;
private final WritableDatabaseInsightIndex relationshipIndex; private final WritableDatabaseBloomIndex relationshipIndex;
private final String[] properties; private final String[] properties;


InsightIndexTransactionEventUpdater( WritableDatabaseInsightIndex nodeIndex, WritableDatabaseInsightIndex relationshipIndex, String[] properties ) BloomIndexTransactionEventUpdater( WritableDatabaseBloomIndex nodeIndex, WritableDatabaseBloomIndex relationshipIndex, String[] properties )
{ {
this.nodeIndex = nodeIndex; this.nodeIndex = nodeIndex;
this.relationshipIndex = relationshipIndex; this.relationshipIndex = relationshipIndex;
Expand Down Expand Up @@ -92,7 +92,7 @@ public void afterCommit( TransactionData data, Object state )
deleteIndexData( data.deletedNodes(), nodeIndex ); deleteIndexData( data.deletedNodes(), nodeIndex );
} }


private void updatePropertyData( Iterable<PropertyEntry<Node>> propertyEntries, Map<Long,Map<String,Object>> state, WritableDatabaseInsightIndex nodeIndex ) private void updatePropertyData( Iterable<PropertyEntry<Node>> propertyEntries, Map<Long,Map<String,Object>> state, WritableDatabaseBloomIndex nodeIndex )
{ {
for ( PropertyEntry<Node> propertyEntry : propertyEntries ) for ( PropertyEntry<Node> propertyEntry : propertyEntries )
{ {
Expand All @@ -103,10 +103,10 @@ private void updatePropertyData( Iterable<PropertyEntry<Node>> propertyEntries,
return; return;
} }


Document document = LuceneInsightDocumentStructure.documentRepresentingProperties( nodeId, allProperties ); Document document = BloomInsightDocumentStructure.documentRepresentingProperties( nodeId, allProperties );
try try
{ {
nodeIndex.getIndexWriter().updateDocument( LuceneInsightDocumentStructure.newTermForChangeOrRemove( nodeId ), document ); nodeIndex.getIndexWriter().updateDocument( BloomInsightDocumentStructure.newTermForChangeOrRemove( nodeId ), document );
} }
catch ( IOException e ) catch ( IOException e )
{ {
Expand All @@ -123,13 +123,13 @@ private void updatePropertyData( Iterable<PropertyEntry<Node>> propertyEntries,
} }
} }


private void deleteIndexData( Iterable<Node> nodes, WritableDatabaseInsightIndex nodeIndex ) private void deleteIndexData( Iterable<Node> nodes, WritableDatabaseBloomIndex nodeIndex )
{ {
for ( Node node : nodes ) for ( Node node : nodes )
{ {
try try
{ {
nodeIndex.getIndexWriter().deleteDocuments( LuceneInsightDocumentStructure.newTermForChangeOrRemove( node.getId() ) ); nodeIndex.getIndexWriter().deleteDocuments( BloomInsightDocumentStructure.newTermForChangeOrRemove( node.getId() ) );
} }
catch ( IOException e ) catch ( IOException e )
{ {
Expand Down
Expand Up @@ -17,7 +17,7 @@
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
package org.neo4j.kernel.api.impl.insight; package org.neo4j.kernel.api.impl.bloom;


import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
Expand All @@ -34,13 +34,13 @@


import static org.apache.lucene.document.Field.Store.YES; import static org.apache.lucene.document.Field.Store.YES;


class LuceneInsightDocumentStructure class BloomInsightDocumentStructure
{ {
static final String ID_KEY = "id"; static final String ID_KEY = "id";


private static final ThreadLocal<DocWithId> perThreadDocument = ThreadLocal.withInitial( DocWithId::new ); private static final ThreadLocal<DocWithId> perThreadDocument = ThreadLocal.withInitial( DocWithId::new );


private LuceneInsightDocumentStructure() private BloomInsightDocumentStructure()
{ {
} }


Expand All @@ -65,7 +65,7 @@ public static long getNodeId( Document from )


static Field encodeValueField( String propertyKey, Value value ) static Field encodeValueField( String propertyKey, Value value )
{ {
InsightFieldEncoding encoding = InsightFieldEncoding.forValue( value ); BloomFieldEncoding encoding = BloomFieldEncoding.forValue( value );
return encoding.encodeField( propertyKey, value ); return encoding.encodeField( propertyKey, value );
} }


Expand Down
Expand Up @@ -17,7 +17,7 @@
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
package org.neo4j.kernel.api.impl.insight; package org.neo4j.kernel.api.impl.bloom;


import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
Expand All @@ -37,12 +37,12 @@


import static java.util.Collections.singletonMap; import static java.util.Collections.singletonMap;


class InsightLuceneIndex extends AbstractLuceneIndex class BloomLuceneIndex extends AbstractLuceneIndex
{ {


private String[] properties; private String[] properties;


InsightLuceneIndex( PartitionedIndexStorage indexStorage, IndexPartitionFactory partitionFactory, BloomLuceneIndex( PartitionedIndexStorage indexStorage, IndexPartitionFactory partitionFactory,
String[] properties ) String[] properties )
{ {
super( indexStorage, partitionFactory ); super( indexStorage, partitionFactory );
Expand All @@ -55,13 +55,13 @@ class InsightLuceneIndex extends AbstractLuceneIndex


private final TaskCoordinator taskCoordinator = new TaskCoordinator( 10, TimeUnit.MILLISECONDS ); private final TaskCoordinator taskCoordinator = new TaskCoordinator( 10, TimeUnit.MILLISECONDS );


public PartitionedInsightIndexWriter getIndexWriter( WritableDatabaseInsightIndex writableIndex ) throws IOException public PartitionedInsightBloomWriter getIndexWriter( WritableDatabaseBloomIndex writableIndex ) throws IOException
{ {
ensureOpen(); ensureOpen();
return new PartitionedInsightIndexWriter( writableIndex ); return new PartitionedInsightBloomWriter( writableIndex );
} }


public InsightIndexReader getIndexReader() throws IOException public BloomIndexReader getIndexReader() throws IOException
{ {
ensureOpen(); ensureOpen();
List<AbstractIndexPartition> partitions = getPartitions(); List<AbstractIndexPartition> partitions = getPartitions();
Expand Down Expand Up @@ -126,17 +126,17 @@ public void markAsFailed( String failure ) throws IOException
indexStorage.storeIndexFailure( failure ); indexStorage.storeIndexFailure( failure );
} }


private SimpleInsightIndexReader createSimpleReader( List<AbstractIndexPartition> partitions ) throws IOException private SimpleBloomIndexReader createSimpleReader( List<AbstractIndexPartition> partitions ) throws IOException
{ {
AbstractIndexPartition singlePartition = getFirstPartition( partitions ); AbstractIndexPartition singlePartition = getFirstPartition( partitions );
return new SimpleInsightIndexReader( singlePartition.acquireSearcher(), properties ); return new SimpleBloomIndexReader( singlePartition.acquireSearcher(), properties );
} }


private PartitionedInsightIndexReader createPartitionedReader( List<AbstractIndexPartition> partitions ) private PartitionedBloomIndexReader createPartitionedReader( List<AbstractIndexPartition> partitions )
throws IOException throws IOException
{ {
List<PartitionSearcher> searchers = acquireSearchers( partitions ); List<PartitionSearcher> searchers = acquireSearchers( partitions );
return new PartitionedInsightIndexReader( searchers, properties ); return new PartitionedBloomIndexReader( searchers, properties );
} }


} }
Expand Up @@ -17,7 +17,7 @@
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
package org.neo4j.kernel.api.impl.insight; package org.neo4j.kernel.api.impl.bloom;


import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
Expand All @@ -26,30 +26,29 @@


import org.neo4j.collection.primitive.PrimitiveLongCollections; import org.neo4j.collection.primitive.PrimitiveLongCollections;
import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.collection.primitive.PrimitiveLongIterator;
import org.neo4j.helpers.TaskCoordinator;
import org.neo4j.io.IOUtils; import org.neo4j.io.IOUtils;
import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher; import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher;
import org.neo4j.kernel.api.impl.schema.reader.IndexReaderCloseException; import org.neo4j.kernel.api.impl.schema.reader.IndexReaderCloseException;


/** /**
* Index reader that is able to read/sample multiple partitions of a partitioned Lucene index. * Index reader that is able to read/sample multiple partitions of a partitioned Lucene index.
* Internally uses multiple {@link SimpleInsightIndexReader}s for individual partitions. * Internally uses multiple {@link SimpleBloomIndexReader}s for individual partitions.
* *
* @see SimpleInsightIndexReader * @see SimpleBloomIndexReader
*/ */
class PartitionedInsightIndexReader implements InsightIndexReader class PartitionedBloomIndexReader implements BloomIndexReader
{ {


private final List<InsightIndexReader> indexReaders; private final List<BloomIndexReader> indexReaders;


PartitionedInsightIndexReader( List<PartitionSearcher> partitionSearchers, String[] properties ) PartitionedBloomIndexReader( List<PartitionSearcher> partitionSearchers, String[] properties )
{ {
this( partitionSearchers.stream().map( partitionSearcher -> new SimpleInsightIndexReader( partitionSearcher, this( partitionSearchers.stream().map( partitionSearcher -> new SimpleBloomIndexReader( partitionSearcher,
properties ) ) properties ) )
.collect( Collectors.toList() ) ); .collect( Collectors.toList() ) );
} }


private PartitionedInsightIndexReader( List<InsightIndexReader> readers ) private PartitionedBloomIndexReader( List<BloomIndexReader> readers )
{ {
this.indexReaders = readers; this.indexReaders = readers;
} }
Expand All @@ -59,7 +58,7 @@ public PrimitiveLongIterator query( String... query )
return partitionedOperation( reader -> innerQuery( reader, query ) ); return partitionedOperation( reader -> innerQuery( reader, query ) );
} }


private PrimitiveLongIterator innerQuery( InsightIndexReader reader, String... query ) private PrimitiveLongIterator innerQuery( BloomIndexReader reader, String... query )
{ {


return reader.query( query ); return reader.query( query );
Expand All @@ -79,7 +78,7 @@ public void close()
} }


private PrimitiveLongIterator partitionedOperation( private PrimitiveLongIterator partitionedOperation(
Function<InsightIndexReader,PrimitiveLongIterator> readerFunction ) Function<BloomIndexReader,PrimitiveLongIterator> readerFunction )
{ {
return PrimitiveLongCollections return PrimitiveLongCollections
.concat( indexReaders.parallelStream().map( readerFunction::apply ).collect( Collectors.toList() ) ); .concat( indexReaders.parallelStream().map( readerFunction::apply ).collect( Collectors.toList() ) );
Expand Down
Expand Up @@ -17,10 +17,9 @@
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
package org.neo4j.kernel.api.impl.insight; package org.neo4j.kernel.api.impl.bloom;


import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
Expand All @@ -29,7 +28,6 @@
import java.util.List; import java.util.List;


import org.neo4j.kernel.api.impl.index.partition.AbstractIndexPartition; import org.neo4j.kernel.api.impl.index.partition.AbstractIndexPartition;
import org.neo4j.kernel.api.impl.schema.writer.LuceneIndexWriter;


/** /**
* Schema Lucene index writer implementation that supports writing into multiple partitions and creates partitions * Schema Lucene index writer implementation that supports writing into multiple partitions and creates partitions
Expand All @@ -39,16 +37,16 @@
* {@link #MAXIMUM_PARTITION_SIZE}. * {@link #MAXIMUM_PARTITION_SIZE}.
* First observable partition that satisfy writer criteria is used for writing. * First observable partition that satisfy writer criteria is used for writing.
*/ */
class PartitionedInsightIndexWriter class PartitionedInsightBloomWriter
{ {
private final WritableDatabaseInsightIndex index; private final WritableDatabaseBloomIndex index;


// by default we still keep a spare of 10% to the maximum partition size: During concurrent updates // by default we still keep a spare of 10% to the maximum partition size: During concurrent updates
// it could happen that 2 threads reserve space in a partition (without claiming it by doing addDocument): // it could happen that 2 threads reserve space in a partition (without claiming it by doing addDocument):
private final Integer MAXIMUM_PARTITION_SIZE = Integer.getInteger( "luceneSchemaIndex.maxPartitionSize", private final Integer MAXIMUM_PARTITION_SIZE = Integer.getInteger( "luceneSchemaIndex.maxPartitionSize",
IndexWriter.MAX_DOCS - (IndexWriter.MAX_DOCS / 10) ); IndexWriter.MAX_DOCS - (IndexWriter.MAX_DOCS / 10) );


PartitionedInsightIndexWriter( WritableDatabaseInsightIndex index ) throws IOException PartitionedInsightBloomWriter( WritableDatabaseBloomIndex index ) throws IOException
{ {
this.index = index; this.index = index;
} }
Expand Down

0 comments on commit 074da37

Please sign in to comment.