Skip to content

Commit

Permalink
Adapted lucene to new index population
Browse files Browse the repository at this point in the history
  • Loading branch information
@fickludd @ragadeeshu
fickludd authored and ragadeeshu committed Feb 15, 2017
1 parent bfd3d70 commit 1b9894b
Show file tree
Hide file tree
Showing 11 changed files with 192 additions and 168 deletions.
14 changes: 7 additions & 7 deletions ...nity/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneIndexAccessor.java
View file
Expand Up @@ -29,8 +29,8 @@
import org.neo4j.kernel.api.impl.schema.reader.LuceneAllEntriesIndexAccessorReader;
import org.neo4j.kernel.api.impl.schema.writer.LuceneIndexWriter;
import org.neo4j.kernel.api.index.IndexAccessor;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyUpdate;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
import org.neo4j.storageengine.api.schema.IndexReader;

Expand Down Expand Up @@ -127,25 +127,25 @@ private LuceneIndexUpdater( LuceneIndexWriter indexWriter, boolean isRecovery )
}

@Override
public void process( NodePropertyUpdate update ) throws IOException
public void process( IndexEntryUpdate update ) throws IOException
{
switch ( update.getUpdateMode() )
switch ( update.updateMode() )
{
case ADDED:
if ( isRecovery )
{
addRecovered( update.getNodeId(), update.getValueAfter() );
addRecovered( update.getEntityId(), update.values()[0] );
}
else
{
add( update.getNodeId(), update.getValueAfter() );
add( update.getEntityId(), update.values()[0] );
}
break;
case CHANGED:
change( update.getNodeId(), update.getValueAfter() );
change( update.getEntityId(), update.values()[0] );
break;
case REMOVED:
remove( update.getNodeId() );
remove( update.getEntityId() );
break;
default:
throw new UnsupportedOperationException();
Expand Down
4 changes: 2 additions & 2 deletions .../src/main/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulator.java
View file
Expand Up @@ -23,9 +23,9 @@

import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
import org.neo4j.kernel.api.impl.schema.SchemaIndex;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.schema.IndexDescriptor;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyUpdate;
import org.neo4j.kernel.api.index.PropertyAccessor;
import org.neo4j.kernel.impl.api.index.sampling.UniqueIndexSampler;
import org.neo4j.storageengine.api.schema.IndexSample;
Expand Down Expand Up @@ -62,7 +62,7 @@ public IndexUpdater newPopulatingUpdater( final PropertyAccessor accessor ) thro
}

@Override
public void includeSample( NodePropertyUpdate update )
public void includeSample( IndexEntryUpdate update )
{
sampler.increment( 1 );
}
Expand Down
9 changes: 6 additions & 3 deletions ...ne-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndexPopulationIT.java
View file
Expand Up @@ -35,8 +35,10 @@
import org.neo4j.kernel.api.impl.schema.LuceneIndexAccessor;
import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexBuilder;
import org.neo4j.kernel.api.impl.schema.SchemaIndex;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyUpdate;
import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptor;
import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptorFactory;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.IndexSample;
Expand All @@ -58,6 +60,7 @@ public class LuceneSchemaIndexPopulationIT
public final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule();

private int affectedNodes;
private NewIndexDescriptor index = NewIndexDescriptorFactory.forLabel( 0, 0 );

@Before
public void before() throws Exception
Expand Down Expand Up @@ -132,8 +135,8 @@ private void generateUpdates( LuceneIndexAccessor indexAccessor, int nodesToUpda
}
}

private NodePropertyUpdate add( long nodeId, Object value )
private IndexEntryUpdate add( long nodeId, Object value )
{
return NodePropertyUpdate.add( nodeId, 0, value, new long[0] );
return IndexEntryUpdate.add( nodeId, index, value );
}
}
21 changes: 12 additions & 9 deletions ...e-index/src/test/java/org/neo4j/kernel/api/impl/schema/AccessUniqueDatabaseIndexTest.java
View file
Expand Up @@ -31,8 +31,10 @@
import org.neo4j.kernel.api.impl.index.storage.IndexStorageFactory;
import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
import org.neo4j.kernel.api.index.IndexAccessor;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyUpdate;
import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptor;
import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptorFactory;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
import org.neo4j.test.rule.fs.EphemeralFileSystemRule;

Expand All @@ -46,6 +48,7 @@ public class AccessUniqueDatabaseIndexTest
public final EphemeralFileSystemRule fileSystemRule = new EphemeralFileSystemRule();
private final DirectoryFactory directoryFactory = new DirectoryFactory.InMemoryDirectoryFactory();
private final File indexDirectory = new File( "index1" );
private final NewIndexDescriptor index = NewIndexDescriptorFactory.forLabel( 1000, 100 );

@Test
public void shouldAddUniqueEntries() throws Exception
Expand Down Expand Up @@ -145,19 +148,19 @@ private PartitionedIndexStorage getIndexStorage() throws IOException
return storageFactory.indexStorageOf( 1, false );
}

private NodePropertyUpdate add( long nodeId, Object propertyValue )
private IndexEntryUpdate add( long nodeId, Object propertyValue )
{
return NodePropertyUpdate.add( nodeId, 100, propertyValue, new long[]{1000} );
return IndexEntryUpdate.add( nodeId, index, propertyValue );
}

private NodePropertyUpdate change( long nodeId, Object oldValue, Object newValue )
private IndexEntryUpdate change( long nodeId, Object oldValue, Object newValue )
{
return NodePropertyUpdate.change( nodeId, 100, oldValue, new long[]{1000}, newValue, new long[]{1000} );
return IndexEntryUpdate.change( nodeId, index, oldValue, newValue );
}

private NodePropertyUpdate remove( long nodeId, Object oldValue )
private IndexEntryUpdate remove( long nodeId, Object oldValue )
{
return NodePropertyUpdate.remove( nodeId, 100, oldValue, new long[]{1000} );
return IndexEntryUpdate.remove( nodeId, index, oldValue );
}

private List<Long> getAllNodes( PartitionedIndexStorage indexStorage, String propertyValue ) throws IOException
Expand All @@ -166,12 +169,12 @@ private List<Long> getAllNodes( PartitionedIndexStorage indexStorage, String pro
propertyValue );
}

private void updateAndCommit( IndexAccessor accessor, Iterable<NodePropertyUpdate> updates )
private void updateAndCommit( IndexAccessor accessor, Iterable<IndexEntryUpdate> updates )
throws IOException, IndexEntryConflictException
{
try ( IndexUpdater updater = accessor.newUpdater( IndexUpdateMode.ONLINE ) )
{
for ( NodePropertyUpdate update : updates )
for ( IndexEntryUpdate update : updates )
{
updater.process( update );
}
Expand Down
21 changes: 12 additions & 9 deletions ...ucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/DatabaseIndexAccessorTest.java
View file
Expand Up @@ -41,8 +41,10 @@
import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyUpdate;
import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptor;
import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptorFactory;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.IndexSampler;
Expand Down Expand Up @@ -73,6 +75,7 @@ public class DatabaseIndexAccessorTest
private final long nodeId = 1, nodeId2 = 2;
private final Object value = "value", value2 = 40;
private DirectoryFactory.InMemoryDirectoryFactory dirFactory;
private NewIndexDescriptor index = NewIndexDescriptorFactory.forLabel( 0, 0 );

@Parameterized.Parameters( name = "{0}" )
public static Collection<IOFunction<DirectoryFactory,LuceneIndexAccessor>[]> implementations()
Expand Down Expand Up @@ -319,27 +322,27 @@ public Void apply( Void nothing ) throws IOException
}
}

private NodePropertyUpdate add( long nodeId, Object value )
private IndexEntryUpdate add( long nodeId, Object value )
{
return NodePropertyUpdate.add( nodeId, 0, value, new long[0] );
return IndexEntryUpdate.add( nodeId, index, value );
}

private NodePropertyUpdate remove( long nodeId, Object value )
private IndexEntryUpdate remove( long nodeId, Object value )
{
return NodePropertyUpdate.remove( nodeId, 0, value, new long[0] );
return IndexEntryUpdate.remove( nodeId, index, value );
}

private NodePropertyUpdate change( long nodeId, Object valueBefore, Object valueAfter )
private IndexEntryUpdate change( long nodeId, Object valueBefore, Object valueAfter )
{
return NodePropertyUpdate.change( nodeId, 0, valueBefore, new long[0], valueAfter, new long[0] );
return IndexEntryUpdate.change( nodeId, index, valueBefore, valueAfter );
}

private void updateAndCommit( List<NodePropertyUpdate> nodePropertyUpdates )
private void updateAndCommit( List<IndexEntryUpdate> nodePropertyUpdates )
throws IOException, IndexEntryConflictException
{
try ( IndexUpdater updater = accessor.newUpdater( IndexUpdateMode.ONLINE ) )
{
for ( NodePropertyUpdate update : nodePropertyUpdates )
for ( IndexEntryUpdate update : nodePropertyUpdates )
{
updater.process( update );
}
Expand Down
10 changes: 7 additions & 3 deletions ...nity/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexIT.java
View file
Expand Up @@ -39,8 +39,10 @@
import org.neo4j.helpers.collection.Iterators;
import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
import org.neo4j.kernel.api.impl.index.LuceneAllDocumentsReader;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.NodePropertyUpdate;
import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptor;
import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptorFactory;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
import org.neo4j.test.rule.TestDirectory;
import org.neo4j.test.rule.fs.DefaultFileSystemRule;
Expand All @@ -61,6 +63,8 @@ public class LuceneSchemaIndexIT
@Rule
public final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule();

private NewIndexDescriptor index = NewIndexDescriptorFactory.forLabel( 0, 0 );

@Before
public void before() throws Exception
{
Expand Down Expand Up @@ -302,9 +306,9 @@ private void generateUpdates( LuceneIndexAccessor indexAccessor, int nodesToUpda
}
}

private NodePropertyUpdate add( long nodeId, Object value )
private IndexEntryUpdate add( long nodeId, Object value )
{
return NodePropertyUpdate.add( nodeId, 0, value, new long[0] );
return IndexEntryUpdate.add( nodeId, index, value );
}

}
32 changes: 16 additions & 16 deletions ...-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexPopulatorTest.java
View file
Expand Up @@ -39,13 +39,14 @@
import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
import org.neo4j.kernel.api.index.IndexConfiguration;
import org.neo4j.kernel.api.schema.IndexDescriptor;
import org.neo4j.kernel.api.schema.IndexDescriptorFactory;
import org.neo4j.kernel.api.index.IndexEntryUpdate;
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.InternalIndexState;
import org.neo4j.kernel.api.index.NodePropertyUpdate;
import org.neo4j.kernel.api.index.PropertyAccessor;
import org.neo4j.kernel.api.schema_new.index.IndexBoundary;
import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptor;
import org.neo4j.kernel.api.schema_new.index.NewIndexDescriptorFactory;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.api.index.IndexStoreView;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
Expand All @@ -67,15 +68,15 @@ public class LuceneSchemaIndexPopulatorTest
@Rule
public TestDirectory testDir = TestDirectory.testDirectory();

private IndexDescriptor indexDescriptor;
private IndexStoreView indexStoreView;
private LuceneSchemaIndexProvider provider;
private Directory directory;
private IndexPopulator indexPopulator;
private IndexReader reader;
private IndexSearcher searcher;
private final long indexId = 0;
private final int propertyKeyId = 666;
private final static int propertyKeyId = 666;
private final static NewIndexDescriptor index = NewIndexDescriptorFactory.forLabel( 42, propertyKeyId );

@Before
public void before() throws Exception
Expand All @@ -85,11 +86,10 @@ public void before() throws Exception
new DirectoryFactory.UncloseableDirectory( directory ) );
provider = new LuceneSchemaIndexProvider( fs.get(), directoryFactory, testDir.directory( "folder" ),
NullLogProvider.getInstance(), Config.empty(), OperationalMode.single );
indexDescriptor = IndexDescriptorFactory.of( 42, propertyKeyId );
indexStoreView = mock( IndexStoreView.class );
IndexConfiguration indexConfig = IndexConfiguration.NON_UNIQUE;
IndexSamplingConfig samplingConfig = new IndexSamplingConfig( Config.empty() );
indexPopulator = provider.getPopulator( indexId, indexDescriptor, indexConfig, samplingConfig );
indexPopulator = provider.getPopulator( indexId, IndexBoundary.map( index ), indexConfig, samplingConfig );
indexPopulator.create();
indexPopulator.configureSampling( true );
}
Expand Down Expand Up @@ -256,19 +256,19 @@ private static class Hit
}
}

private NodePropertyUpdate add( long nodeId, Object value )
private IndexEntryUpdate add( long nodeId, Object value )
{
return NodePropertyUpdate.add( nodeId, 0, value, new long[0] );
return IndexEntryUpdate.add( nodeId, index, value );
}

private NodePropertyUpdate change( long nodeId, Object valueBefore, Object valueAfter )
private IndexEntryUpdate change( long nodeId, Object valueBefore, Object valueAfter )
{
return NodePropertyUpdate.change( nodeId, 0, valueBefore, new long[0], valueAfter, new long[0] );
return IndexEntryUpdate.change( nodeId, index, valueBefore, valueAfter );
}

private NodePropertyUpdate remove( long nodeId, Object removedValue )
private IndexEntryUpdate remove( long nodeId, Object removedValue )
{
return NodePropertyUpdate.remove( nodeId, 0, removedValue, new long[0] );
return IndexEntryUpdate.remove( nodeId, index, removedValue );
}

private void assertIndexedValues( Hit... expectedHits ) throws IOException
Expand Down Expand Up @@ -300,18 +300,18 @@ private void switchToVerification() throws IOException
private static void addUpdate( IndexPopulator populator, long nodeId, Object value )
throws IOException, IndexEntryConflictException
{
populator.add( Collections.singletonList( NodePropertyUpdate.add( nodeId, 0, value, new long[]{0} ) ) );
populator.add( Collections.singletonList( IndexEntryUpdate.add( nodeId, index, value ) ) );
}

private static void updatePopulator(
IndexPopulator populator,
Iterable<NodePropertyUpdate> updates,
Iterable<IndexEntryUpdate> updates,
PropertyAccessor accessor )
throws IOException, IndexEntryConflictException
{
try ( IndexUpdater updater = populator.newPopulatingUpdater( accessor ) )
{
for ( NodePropertyUpdate update : updates )
for ( IndexEntryUpdate update : updates )
{
updater.process( update );
}
Expand Down

0 comments on commit 1b9894b

Please sign in to comment.