diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulatingUpdater.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulatingUpdater.java new file mode 100644 index 000000000000..5f565f9d23e9 --- /dev/null +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulatingUpdater.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api.impl.schema.populator; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.Term; + +import java.io.IOException; + +import org.neo4j.collection.primitive.PrimitiveLongSet; +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; +import org.neo4j.kernel.api.impl.schema.LuceneDocumentStructure; +import org.neo4j.kernel.api.impl.schema.writer.LuceneIndexWriter; +import org.neo4j.kernel.api.index.IndexUpdater; +import org.neo4j.kernel.api.index.NodePropertyUpdate; +import org.neo4j.kernel.impl.api.index.UpdateMode; + +/** + * An {@link IndexUpdater} used while index population is in progress. Takes special care of node property additions + * and changes applying them via {@link LuceneIndexWriter#updateDocument(Term, Document)} to make sure no duplicated + * documents are inserted. + */ +public abstract class LuceneIndexPopulatingUpdater implements IndexUpdater +{ + private final LuceneIndexWriter writer; + + public LuceneIndexPopulatingUpdater( LuceneIndexWriter writer ) + { + this.writer = writer; + } + + @Override + public void process( NodePropertyUpdate update ) throws IOException, IndexEntryConflictException + { + long nodeId = update.getNodeId(); + + switch ( update.getUpdateMode() ) + { + case ADDED: + added( update ); + writer.updateDocument( LuceneDocumentStructure.newTermForChangeOrRemove( nodeId ), + LuceneDocumentStructure.documentRepresentingProperty( nodeId, update.getValueAfter() ) ); + break; + case CHANGED: + changed( update ); + writer.updateDocument( LuceneDocumentStructure.newTermForChangeOrRemove( nodeId ), + LuceneDocumentStructure.documentRepresentingProperty( nodeId, update.getValueAfter() ) ); + break; + case REMOVED: + removed( update ); + writer.deleteDocuments( LuceneDocumentStructure.newTermForChangeOrRemove( nodeId ) ); + break; + default: + throw new IllegalStateException( "Unknown update mode " + update.getUpdateMode() ); + } + } + + @Override + public final void remove( PrimitiveLongSet nodeIds ) + { + throw new UnsupportedOperationException( "Should not remove from populating index" ); + } + + /** + * Method is invoked when {@link NodePropertyUpdate} with {@link UpdateMode#ADDED} is processed. + * + * @param update the update being processed. + */ + protected abstract void added( NodePropertyUpdate update ); + + /** + * Method is invoked when {@link NodePropertyUpdate} with {@link UpdateMode#CHANGED} is processed. + * + * @param update the update being processed. + */ + protected abstract void changed( NodePropertyUpdate update ); + + /** + * Method is invoked when {@link NodePropertyUpdate} with {@link UpdateMode#REMOVED} is processed. + * + * @param update the update being processed. + */ + protected abstract void removed( NodePropertyUpdate update ); +} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulator.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulator.java index 6754c32ea21a..179282d30536 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulator.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/LuceneIndexPopulator.java @@ -25,6 +25,7 @@ import java.util.Iterator; import java.util.List; +import org.neo4j.io.IOUtils; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.impl.schema.LuceneDocumentStructure; import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndex; @@ -32,6 +33,9 @@ import org.neo4j.kernel.api.index.IndexPopulator; import org.neo4j.kernel.api.index.NodePropertyUpdate; +/** + * An {@link IndexPopulator} used to create, populate and mark as online a Lucene schema index. + */ public abstract class LuceneIndexPopulator implements IndexPopulator { protected LuceneSchemaIndex luceneIndex; @@ -75,13 +79,12 @@ public void close( boolean populationCompletedSuccessfully ) throws IOException { if ( populationCompletedSuccessfully ) { - flush(); luceneIndex.markAsOnline(); } } finally { - luceneIndex.close(); + IOUtils.closeAllSilently( luceneIndex ); } } @@ -91,8 +94,6 @@ public void markAsFailed( String failure ) throws IOException luceneIndex.markAsFailed( failure ); } - protected abstract void flush() throws IOException; - private static Document updateAsDocument( NodePropertyUpdate update ) { return LuceneDocumentStructure.documentRepresentingProperty( update.getNodeId(), update.getValueAfter() ); diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulatingUpdater.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulatingUpdater.java new file mode 100644 index 000000000000..6df52214289c --- /dev/null +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulatingUpdater.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api.impl.schema.populator; + +import org.neo4j.kernel.api.impl.schema.LuceneDocumentStructure; +import org.neo4j.kernel.api.impl.schema.writer.LuceneIndexWriter; +import org.neo4j.kernel.api.index.NodePropertyUpdate; +import org.neo4j.kernel.impl.api.index.sampling.NonUniqueIndexSampler; + +/** + * A {@link LuceneIndexPopulatingUpdater} used for non-unique Lucene schema indexes. + */ +public class NonUniqueLuceneIndexPopulatingUpdater extends LuceneIndexPopulatingUpdater +{ + private final NonUniqueIndexSampler sampler; + + public NonUniqueLuceneIndexPopulatingUpdater( LuceneIndexWriter writer, NonUniqueIndexSampler sampler ) + { + super( writer ); + this.sampler = sampler; + } + + @Override + protected void added( NodePropertyUpdate update ) + { + String encodedValue = LuceneDocumentStructure.encodedStringValue( update.getValueAfter() ); + sampler.include( encodedValue ); + } + + @Override + protected void changed( NodePropertyUpdate update ) + { + String encodedValueBefore = LuceneDocumentStructure.encodedStringValue( update.getValueBefore() ); + sampler.exclude( encodedValueBefore ); + + String encodedValueAfter = LuceneDocumentStructure.encodedStringValue( update.getValueAfter() ); + sampler.include( encodedValueAfter ); + } + + @Override + protected void removed( NodePropertyUpdate update ) + { + String removedValue = LuceneDocumentStructure.encodedStringValue( update.getValueBefore() ); + sampler.exclude( removedValue ); + } + + @Override + public void close() + { + } +} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulator.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulator.java index 445c72dfdc73..41fae773573d 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulator.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulator.java @@ -20,10 +20,7 @@ package org.neo4j.kernel.api.impl.schema.populator; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import org.neo4j.collection.primitive.PrimitiveLongSet; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.impl.schema.LuceneDocumentStructure; import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndex; @@ -33,14 +30,14 @@ import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; import org.neo4j.kernel.impl.api.index.sampling.NonUniqueIndexSampler; import org.neo4j.storageengine.api.schema.IndexSample; -import org.neo4j.unsafe.impl.internal.dragons.FeatureToggles; +/** + * A {@link LuceneIndexPopulator} used for non-unique Lucene schema indexes. + * Performs sampling using {@link NonUniqueIndexSampler}. + */ public class NonUniqueLuceneIndexPopulator extends LuceneIndexPopulator { - private final int queueThreshold = FeatureToggles.getInteger( NonUniqueLuceneIndexPopulator.class, - "queueThreshold", 10000 ); private final NonUniqueIndexSampler sampler; - private final List updates = new ArrayList<>(); public NonUniqueLuceneIndexPopulator( LuceneSchemaIndex luceneIndex, IndexSamplingConfig samplingConfig ) { @@ -57,53 +54,7 @@ public void verifyDeferredConstraints( PropertyAccessor accessor ) throws IndexE @Override public IndexUpdater newPopulatingUpdater( PropertyAccessor propertyAccessor ) throws IOException { - return new IndexUpdater() - { - @Override - public void process( NodePropertyUpdate update ) throws IOException, IndexEntryConflictException - { - switch ( update.getUpdateMode() ) - { - case ADDED: - // We don't look at the "before" value, so adding and changing idempotently is done the same way. - String encodedValue = LuceneDocumentStructure.encodedStringValue( update.getValueAfter() ); - sampler.include( encodedValue ); - break; - case CHANGED: - // We don't look at the "before" value, so adding and changing idempotently is done the same way. - String encodedValueBefore = LuceneDocumentStructure.encodedStringValue( update.getValueBefore() ); - sampler.exclude( encodedValueBefore ); - String encodedValueAfter = LuceneDocumentStructure.encodedStringValue( update.getValueAfter() ); - sampler.include( encodedValueAfter ); - break; - case REMOVED: - String removedValue = LuceneDocumentStructure.encodedStringValue( update.getValueBefore() ); - sampler.exclude( removedValue ); - break; - default: - throw new IllegalStateException( "Unknown update mode " + update.getUpdateMode() ); - } - - updates.add( update ); - } - - @Override - public void close() throws IOException, IndexEntryConflictException - { - if ( updates.size() > queueThreshold ) - { - flush(); - updates.clear(); - } - - } - - @Override - public void remove( PrimitiveLongSet nodeIds ) throws IOException - { - throw new UnsupportedOperationException( "Should not remove() from populating index." ); - } - }; + return new NonUniqueLuceneIndexPopulatingUpdater( writer, sampler ); } @Override @@ -117,27 +68,4 @@ public IndexSample sampleResult() { return sampler.result(); } - - @Override - protected void flush() throws IOException - { - for ( NodePropertyUpdate update : this.updates ) - { - long nodeId = update.getNodeId(); - switch ( update.getUpdateMode() ) - { - case ADDED: - case CHANGED: - // We don't look at the "before" value, so adding and changing idempotently is done the same way. - writer.updateDocument( LuceneDocumentStructure.newTermForChangeOrRemove( nodeId ), - LuceneDocumentStructure.documentRepresentingProperty( nodeId, update.getValueAfter() ) ); - break; - case REMOVED: - writer.deleteDocuments( LuceneDocumentStructure.newTermForChangeOrRemove( nodeId ) ); - break; - default: - throw new IllegalStateException( "Unknown update mode " + update.getUpdateMode() ); - } - } - } } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulatingUpdater.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulatingUpdater.java new file mode 100644 index 000000000000..94448c2d0d95 --- /dev/null +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulatingUpdater.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api.impl.schema.populator; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; +import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndex; +import org.neo4j.kernel.api.impl.schema.writer.LuceneIndexWriter; +import org.neo4j.kernel.api.index.NodePropertyUpdate; +import org.neo4j.kernel.api.index.PropertyAccessor; +import org.neo4j.kernel.impl.api.index.sampling.UniqueIndexSampler; + +/** + * A {@link LuceneIndexPopulatingUpdater} used for unique Lucene schema indexes. + * Verifies uniqueness of added and changed values when closed using + * {@link LuceneSchemaIndex#verifyUniqueness(PropertyAccessor, int, List)} method. + */ +public class UniqueLuceneIndexPopulatingUpdater extends LuceneIndexPopulatingUpdater +{ + private final int propertyKeyId; + private final LuceneSchemaIndex luceneIndex; + private final PropertyAccessor propertyAccessor; + private final UniqueIndexSampler sampler; + + private final List updatedPropertyValues = new ArrayList<>(); + + public UniqueLuceneIndexPopulatingUpdater( LuceneIndexWriter writer, int propertyKeyId, + LuceneSchemaIndex luceneIndex, PropertyAccessor propertyAccessor, UniqueIndexSampler sampler ) + { + super( writer ); + this.propertyKeyId = propertyKeyId; + this.luceneIndex = luceneIndex; + this.propertyAccessor = propertyAccessor; + this.sampler = sampler; + } + + @Override + protected void added( NodePropertyUpdate update ) + { + sampler.increment( 1 ); + updatedPropertyValues.add( update.getValueAfter() ); + } + + @Override + protected void changed( NodePropertyUpdate update ) + { + // sampler.increment( -1 ); // remove old vale + // sampler.increment( 1 ); // add new value + + updatedPropertyValues.add( update.getValueAfter() ); + } + + @Override + protected void removed( NodePropertyUpdate update ) + { + sampler.increment( -1 ); + } + + @Override + public void close() throws IOException, IndexEntryConflictException + { + luceneIndex.verifyUniqueness( propertyAccessor, propertyKeyId, updatedPropertyValues ); + } +} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulator.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulator.java index ab19a037f125..cb5da27953ca 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulator.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulator.java @@ -20,12 +20,8 @@ package org.neo4j.kernel.api.impl.schema.populator; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import org.neo4j.collection.primitive.PrimitiveLongSet; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; -import org.neo4j.kernel.api.impl.schema.LuceneDocumentStructure; import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndex; import org.neo4j.kernel.api.index.IndexDescriptor; import org.neo4j.kernel.api.index.IndexUpdater; @@ -34,81 +30,34 @@ import org.neo4j.kernel.impl.api.index.sampling.UniqueIndexSampler; import org.neo4j.storageengine.api.schema.IndexSample; +/** + * A {@link LuceneIndexPopulator} used for unique Lucene schema indexes. + * Performs sampling using {@link UniqueIndexSampler}. + * Verifies uniqueness of added and changed values using + * {@link LuceneSchemaIndex#verifyUniqueness(PropertyAccessor, int)} method. + */ public class UniqueLuceneIndexPopulator extends LuceneIndexPopulator { - private final IndexDescriptor descriptor; + private final int propertyKeyId; private final UniqueIndexSampler sampler; - public UniqueLuceneIndexPopulator(LuceneSchemaIndex index, IndexDescriptor descriptor ) + public UniqueLuceneIndexPopulator( LuceneSchemaIndex index, IndexDescriptor descriptor ) { super( index ); - this.descriptor = descriptor; + this.propertyKeyId = descriptor.getPropertyKeyId(); this.sampler = new UniqueIndexSampler(); } - @Override - protected void flush() throws IOException - { - // no need to do anything yet. - } - @Override public void verifyDeferredConstraints( PropertyAccessor accessor ) throws IndexEntryConflictException, IOException { - luceneIndex.verifyUniqueness( accessor, descriptor.getPropertyKeyId() ); + luceneIndex.verifyUniqueness( accessor, propertyKeyId ); } @Override public IndexUpdater newPopulatingUpdater( final PropertyAccessor accessor ) throws IOException { - return new IndexUpdater() - { - List updatedPropertyValues = new ArrayList<>(); - - @Override - public void process( NodePropertyUpdate update ) throws IOException, IndexEntryConflictException - { - long nodeId = update.getNodeId(); - switch ( update.getUpdateMode() ) - { - case ADDED: - sampler.increment( 1 ); // add new value - - // We don't look at the "before" value, so adding and changing idempotently is done the same way. - writer.updateDocument( LuceneDocumentStructure.newTermForChangeOrRemove( nodeId ), - LuceneDocumentStructure.documentRepresentingProperty( nodeId, update.getValueAfter() ) ); - updatedPropertyValues.add( update.getValueAfter() ); - break; - case CHANGED: - // sampler.increment( -1 ); // remove old vale - // sampler.increment( 1 ); // add new value - - // We don't look at the "before" value, so adding and changing idempotently is done the same way. - writer.updateDocument( LuceneDocumentStructure.newTermForChangeOrRemove( nodeId ), - LuceneDocumentStructure.documentRepresentingProperty( nodeId, update.getValueAfter() ) ); - updatedPropertyValues.add( update.getValueAfter() ); - break; - case REMOVED: - sampler.increment( -1 ); // remove old value - writer.deleteDocuments( LuceneDocumentStructure.newTermForChangeOrRemove( nodeId ) ); - break; - default: - throw new IllegalStateException( "Unknown update mode " + update.getUpdateMode() ); - } - } - - @Override - public void close() throws IOException, IndexEntryConflictException - { - luceneIndex.verifyUniqueness( accessor, descriptor.getPropertyKeyId(), updatedPropertyValues ); - } - - @Override - public void remove( PrimitiveLongSet nodeIds ) - { - throw new UnsupportedOperationException( "should not remove() from populating index" ); - } - }; + return new UniqueLuceneIndexPopulatingUpdater( writer, propertyKeyId, luceneIndex, accessor, sampler ); } @Override diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulatingUpdaterTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulatingUpdaterTest.java new file mode 100644 index 000000000000..f530a7febba4 --- /dev/null +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulatingUpdaterTest.java @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api.impl.schema.populator; + +import org.junit.Test; + +import org.neo4j.collection.primitive.PrimitiveLongCollections; +import org.neo4j.kernel.api.impl.schema.writer.LuceneIndexWriter; +import org.neo4j.kernel.impl.api.index.sampling.NonUniqueIndexSampler; +import org.neo4j.storageengine.api.schema.IndexSample; + +import static org.hamcrest.Matchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.neo4j.kernel.api.impl.schema.LuceneDocumentStructure.documentRepresentingProperty; +import static org.neo4j.kernel.api.impl.schema.LuceneDocumentStructure.newTermForChangeOrRemove; +import static org.neo4j.kernel.api.index.NodePropertyUpdate.add; +import static org.neo4j.kernel.api.index.NodePropertyUpdate.change; +import static org.neo4j.kernel.api.index.NodePropertyUpdate.remove; + +public class NonUniqueLuceneIndexPopulatingUpdaterTest +{ + private static final int PROPERTY_KEY = 42; + private static final int SAMPLING_BUFFER_SIZE_LIMIT = 100; + + @Test + public void removeNotSupported() + { + NonUniqueLuceneIndexPopulatingUpdater updater = newUpdater(); + + try + { + updater.remove( PrimitiveLongCollections.setOf( 1, 2, 3 ) ); + fail( "Exception expected" ); + } + catch ( Exception e ) + { + assertThat( e, instanceOf( UnsupportedOperationException.class ) ); + } + } + + @Test + public void addedNodePropertiesIncludedInSample() throws Exception + { + NonUniqueIndexSampler sampler = newSampler(); + NonUniqueLuceneIndexPopulatingUpdater updater = newUpdater( sampler ); + + updater.process( add( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( add( 2, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( add( 3, PROPERTY_KEY, "baz", new long[]{1} ) ); + updater.process( add( 4, PROPERTY_KEY, "bar", new long[]{1} ) ); + + verifySamplingResult( sampler, 4, 3, 4 ); + } + + @Test + public void changedNodePropertiesIncludedInSample() throws Exception + { + NonUniqueIndexSampler sampler = newSampler(); + NonUniqueLuceneIndexPopulatingUpdater updater = newUpdater( sampler ); + + updater.process( add( 1, PROPERTY_KEY, "initial1", new long[]{1} ) ); + updater.process( add( 2, PROPERTY_KEY, "initial2", new long[]{1} ) ); + updater.process( add( 3, PROPERTY_KEY, "new2", new long[]{1} ) ); + + updater.process( change( 1, PROPERTY_KEY, "initial1", new long[]{1}, "new1", new long[]{1} ) ); + updater.process( change( 1, PROPERTY_KEY, "initial2", new long[]{1}, "new2", new long[]{1} ) ); + + verifySamplingResult( sampler, 3, 2, 3 ); + } + + @Test + public void removedNodePropertyIncludedInSample() throws Exception + { + NonUniqueIndexSampler sampler = newSampler(); + NonUniqueLuceneIndexPopulatingUpdater updater = newUpdater( sampler ); + + updater.process( add( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( add( 2, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( add( 3, PROPERTY_KEY, "baz", new long[]{1} ) ); + updater.process( add( 4, PROPERTY_KEY, "qux", new long[]{1} ) ); + + updater.process( remove( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( remove( 2, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( remove( 4, PROPERTY_KEY, "qux", new long[]{1} ) ); + + verifySamplingResult( sampler, 1, 1, 1 ); + } + + @Test + public void nodePropertyUpdatesIncludedInSample() throws Exception + { + NonUniqueIndexSampler sampler = newSampler(); + NonUniqueLuceneIndexPopulatingUpdater updater = newUpdater( sampler ); + + updater.process( add( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( change( 1, PROPERTY_KEY, "foo", new long[]{1}, "newFoo1", new long[]{1} ) ); + + updater.process( add( 2, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( remove( 2, PROPERTY_KEY, "bar", new long[]{1} ) ); + + updater.process( change( 1, PROPERTY_KEY, "newFoo1", new long[]{1}, "newFoo2", new long[]{1} ) ); + + updater.process( add( 42, PROPERTY_KEY, "qux", new long[]{1} ) ); + updater.process( add( 3, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( add( 4, PROPERTY_KEY, "baz", new long[]{1} ) ); + updater.process( add( 5, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( remove( 42, PROPERTY_KEY, "qux", new long[]{1} ) ); + + verifySamplingResult( sampler, 4, 3, 4 ); + } + + @Test + public void additionsDeliveredToIndexWriter() throws Exception + { + LuceneIndexWriter writer = mock( LuceneIndexWriter.class ); + NonUniqueLuceneIndexPopulatingUpdater updater = newUpdater( writer ); + + updater.process( add( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( add( 2, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( add( 3, PROPERTY_KEY, "qux", new long[]{1} ) ); + + verify( writer ).updateDocument( newTermForChangeOrRemove( 1 ), documentRepresentingProperty( 1, "foo" ) ); + verify( writer ).updateDocument( newTermForChangeOrRemove( 2 ), documentRepresentingProperty( 2, "bar" ) ); + verify( writer ).updateDocument( newTermForChangeOrRemove( 3 ), documentRepresentingProperty( 3, "qux" ) ); + } + + @Test + public void changesDeliveredToIndexWriter() throws Exception + { + LuceneIndexWriter writer = mock( LuceneIndexWriter.class ); + NonUniqueLuceneIndexPopulatingUpdater updater = newUpdater( writer ); + + updater.process( change( 1, PROPERTY_KEY, "before1", new long[]{1}, "after1", new long[]{1} ) ); + updater.process( change( 2, PROPERTY_KEY, "before2", new long[]{1}, "after2", new long[]{1} ) ); + + verify( writer ).updateDocument( newTermForChangeOrRemove( 1 ), documentRepresentingProperty( 1, "after1" ) ); + verify( writer ).updateDocument( newTermForChangeOrRemove( 2 ), documentRepresentingProperty( 2, "after2" ) ); + } + + @Test + public void removalsDeliveredToIndexWriter() throws Exception + { + LuceneIndexWriter writer = mock( LuceneIndexWriter.class ); + NonUniqueLuceneIndexPopulatingUpdater updater = newUpdater( writer ); + + updater.process( remove( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( remove( 2, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( remove( 3, PROPERTY_KEY, "baz", new long[]{1} ) ); + + verify( writer ).deleteDocuments( newTermForChangeOrRemove( 1 ) ); + verify( writer ).deleteDocuments( newTermForChangeOrRemove( 2 ) ); + verify( writer ).deleteDocuments( newTermForChangeOrRemove( 3 ) ); + } + + + private static void verifySamplingResult( NonUniqueIndexSampler sampler, long expectedIndexSize, + long expectedUniqueValues, long expectedSampleSize ) + { + IndexSample sample = sampler.result(); + + assertEquals( expectedIndexSize, sample.indexSize() ); + assertEquals( expectedUniqueValues, sample.uniqueValues() ); + assertEquals( expectedSampleSize, sample.sampleSize() ); + } + + private static NonUniqueLuceneIndexPopulatingUpdater newUpdater() + { + return newUpdater( newSampler() ); + } + + private static NonUniqueLuceneIndexPopulatingUpdater newUpdater( NonUniqueIndexSampler sampler ) + { + return newUpdater( mock( LuceneIndexWriter.class ), sampler ); + } + + private static NonUniqueLuceneIndexPopulatingUpdater newUpdater( LuceneIndexWriter writer ) + { + return newUpdater( writer, newSampler() ); + } + + private static NonUniqueLuceneIndexPopulatingUpdater newUpdater( LuceneIndexWriter writer, + NonUniqueIndexSampler sampler ) + { + return new NonUniqueLuceneIndexPopulatingUpdater( writer, sampler ); + } + + private static NonUniqueIndexSampler newSampler() + { + return new NonUniqueIndexSampler( SAMPLING_BUFFER_SIZE_LIMIT ); + } +} diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulatingUpdaterTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulatingUpdaterTest.java new file mode 100644 index 000000000000..192be96bee98 --- /dev/null +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/populator/UniqueLuceneIndexPopulatingUpdaterTest.java @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api.impl.schema.populator; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import org.neo4j.collection.primitive.PrimitiveLongCollections; +import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndex; +import org.neo4j.kernel.api.impl.schema.writer.LuceneIndexWriter; +import org.neo4j.kernel.api.index.PropertyAccessor; +import org.neo4j.kernel.impl.api.index.sampling.UniqueIndexSampler; +import org.neo4j.storageengine.api.schema.IndexSample; + +import static org.hamcrest.Matchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.neo4j.kernel.api.impl.schema.LuceneDocumentStructure.documentRepresentingProperty; +import static org.neo4j.kernel.api.impl.schema.LuceneDocumentStructure.newTermForChangeOrRemove; +import static org.neo4j.kernel.api.index.NodePropertyUpdate.add; +import static org.neo4j.kernel.api.index.NodePropertyUpdate.change; +import static org.neo4j.kernel.api.index.NodePropertyUpdate.remove; + +public class UniqueLuceneIndexPopulatingUpdaterTest +{ + private static final int PROPERTY_KEY = 42; + + @Test + public void removeNotSupported() + { + UniqueLuceneIndexPopulatingUpdater updater = newUpdater(); + + try + { + updater.remove( PrimitiveLongCollections.setOf( 1, 2, 3 ) ); + fail( "Exception expected" ); + } + catch ( Exception e ) + { + assertThat( e, instanceOf( UnsupportedOperationException.class ) ); + } + } + + @Test + public void closeVerifiesUniquenessOfAddedValues() throws Exception + { + LuceneSchemaIndex index = mock( LuceneSchemaIndex.class ); + UniqueLuceneIndexPopulatingUpdater updater = newUpdater( index ); + + updater.process( add( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( add( 1, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( add( 1, PROPERTY_KEY, "baz", new long[]{1} ) ); + + verifyZeroInteractions( index ); + + updater.close(); + verify( index ).verifyUniqueness( any(), eq( PROPERTY_KEY ), eq( Arrays.asList( "foo", "bar", "baz" ) ) ); + } + + @Test + public void closeVerifiesUniquenessOfChangedValues() throws Exception + { + LuceneSchemaIndex index = mock( LuceneSchemaIndex.class ); + UniqueLuceneIndexPopulatingUpdater updater = newUpdater( index ); + + updater.process( change( 1, PROPERTY_KEY, "foo1", new long[]{1, 2}, "foo2", new long[]{1} ) ); + updater.process( change( 1, PROPERTY_KEY, "bar1", new long[]{1, 2}, "bar2", new long[]{1} ) ); + updater.process( change( 1, PROPERTY_KEY, "baz1", new long[]{1, 2}, "baz2", new long[]{1} ) ); + + verifyZeroInteractions( index ); + + updater.close(); + verify( index ).verifyUniqueness( any(), eq( PROPERTY_KEY ), eq( Arrays.asList( "foo2", "bar2", "baz2" ) ) ); + } + + @Test + public void closeVerifiesUniquenessOfAddedAndChangedValues() throws Exception + { + LuceneSchemaIndex index = mock( LuceneSchemaIndex.class ); + UniqueLuceneIndexPopulatingUpdater updater = newUpdater( index ); + + updater.process( add( 1, PROPERTY_KEY, "added1", new long[]{1} ) ); + updater.process( add( 2, PROPERTY_KEY, "added2", new long[]{1} ) ); + updater.process( change( 3, PROPERTY_KEY, "before1", new long[]{1, 2}, "after1", new long[]{2} ) ); + updater.process( change( 4, PROPERTY_KEY, "before2", new long[]{1}, "after2", new long[]{1, 2, 3} ) ); + updater.process( remove( 5, PROPERTY_KEY, "removed1", new long[]{1, 2} ) ); + + verifyZeroInteractions( index ); + + updater.close(); + + List toBeVerified = Arrays.asList( "added1", "added2", "after1", "after2" ); + verify( index ).verifyUniqueness( any(), eq( PROPERTY_KEY ), eq( toBeVerified ) ); + } + + @Test + public void addedNodePropertiesIncludedInSample() throws Exception + { + UniqueIndexSampler sampler = new UniqueIndexSampler(); + UniqueLuceneIndexPopulatingUpdater updater = newUpdater( sampler ); + + updater.process( add( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( add( 2, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( add( 3, PROPERTY_KEY, "baz", new long[]{1} ) ); + updater.process( add( 4, PROPERTY_KEY, "qux", new long[]{1} ) ); + + verifySamplingResult( sampler, 4 ); + } + + @Test + public void changedNodePropertiesDoNotInfluenceSample() throws Exception + { + UniqueIndexSampler sampler = new UniqueIndexSampler(); + UniqueLuceneIndexPopulatingUpdater updater = newUpdater( sampler ); + + updater.process( change( 1, PROPERTY_KEY, "before1", new long[]{1}, "after1", new long[]{1} ) ); + updater.process( change( 2, PROPERTY_KEY, "before2", new long[]{1}, "after2", new long[]{1} ) ); + + verifySamplingResult( sampler, 0 ); + } + + @Test + public void removedNodePropertyIncludedInSample() throws Exception + { + long initialValue = 10; + UniqueIndexSampler sampler = new UniqueIndexSampler(); + sampler.increment( initialValue ); + + UniqueLuceneIndexPopulatingUpdater updater = newUpdater( sampler ); + + updater.process( remove( 1, PROPERTY_KEY, "removed1", new long[]{1} ) ); + updater.process( remove( 2, PROPERTY_KEY, "removed2", new long[]{1} ) ); + + verifySamplingResult( sampler, initialValue - 2 ); + } + + @Test + public void nodePropertyUpdatesIncludedInSample() throws Exception + { + UniqueIndexSampler sampler = new UniqueIndexSampler(); + UniqueLuceneIndexPopulatingUpdater updater = newUpdater( sampler ); + + updater.process( add( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( change( 1, PROPERTY_KEY, "foo", new long[]{1}, "bar", new long[]{1} ) ); + updater.process( add( 2, PROPERTY_KEY, "baz", new long[]{1} ) ); + updater.process( add( 3, PROPERTY_KEY, "qux", new long[]{1} ) ); + updater.process( remove( 4, PROPERTY_KEY, "qux", new long[]{1} ) ); + + verifySamplingResult( sampler, 2 ); + } + + @Test + public void additionsDeliveredToIndexWriter() throws Exception + { + LuceneIndexWriter writer = mock( LuceneIndexWriter.class ); + UniqueLuceneIndexPopulatingUpdater updater = newUpdater( writer ); + + updater.process( add( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( add( 2, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( add( 3, PROPERTY_KEY, "qux", new long[]{1} ) ); + + verify( writer ).updateDocument( newTermForChangeOrRemove( 1 ), documentRepresentingProperty( 1, "foo" ) ); + verify( writer ).updateDocument( newTermForChangeOrRemove( 2 ), documentRepresentingProperty( 2, "bar" ) ); + verify( writer ).updateDocument( newTermForChangeOrRemove( 3 ), documentRepresentingProperty( 3, "qux" ) ); + } + + @Test + public void changesDeliveredToIndexWriter() throws Exception + { + LuceneIndexWriter writer = mock( LuceneIndexWriter.class ); + UniqueLuceneIndexPopulatingUpdater updater = newUpdater( writer ); + + updater.process( change( 1, PROPERTY_KEY, "before1", new long[]{1}, "after1", new long[]{1} ) ); + updater.process( change( 2, PROPERTY_KEY, "before2", new long[]{1}, "after2", new long[]{1} ) ); + + verify( writer ).updateDocument( newTermForChangeOrRemove( 1 ), documentRepresentingProperty( 1, "after1" ) ); + verify( writer ).updateDocument( newTermForChangeOrRemove( 2 ), documentRepresentingProperty( 2, "after2" ) ); + } + + @Test + public void removalsDeliveredToIndexWriter() throws Exception + { + LuceneIndexWriter writer = mock( LuceneIndexWriter.class ); + UniqueLuceneIndexPopulatingUpdater updater = newUpdater( writer ); + + updater.process( remove( 1, PROPERTY_KEY, "foo", new long[]{1} ) ); + updater.process( remove( 2, PROPERTY_KEY, "bar", new long[]{1} ) ); + updater.process( remove( 3, PROPERTY_KEY, "baz", new long[]{1} ) ); + + verify( writer ).deleteDocuments( newTermForChangeOrRemove( 1 ) ); + verify( writer ).deleteDocuments( newTermForChangeOrRemove( 2 ) ); + verify( writer ).deleteDocuments( newTermForChangeOrRemove( 3 ) ); + } + + private static void verifySamplingResult( UniqueIndexSampler sampler, long expectedValue ) + { + IndexSample sample = sampler.result(); + + assertEquals( expectedValue, sample.indexSize() ); + assertEquals( expectedValue, sample.uniqueValues() ); + assertEquals( expectedValue, sample.sampleSize() ); + } + + private static UniqueLuceneIndexPopulatingUpdater newUpdater() + { + return newUpdater( new UniqueIndexSampler() ); + } + + private static UniqueLuceneIndexPopulatingUpdater newUpdater( LuceneSchemaIndex index ) + { + return newUpdater( index, mock( LuceneIndexWriter.class ), new UniqueIndexSampler() ); + } + + private static UniqueLuceneIndexPopulatingUpdater newUpdater( LuceneIndexWriter writer ) + { + return newUpdater( mock( LuceneSchemaIndex.class ), writer, new UniqueIndexSampler() ); + } + + private static UniqueLuceneIndexPopulatingUpdater newUpdater( UniqueIndexSampler sampler ) + { + return newUpdater( mock( LuceneSchemaIndex.class ), mock( LuceneIndexWriter.class ), sampler ); + } + + private static UniqueLuceneIndexPopulatingUpdater newUpdater( LuceneSchemaIndex index, LuceneIndexWriter writer, + UniqueIndexSampler sampler ) + { + return new UniqueLuceneIndexPopulatingUpdater( writer, PROPERTY_KEY, index, mock( PropertyAccessor.class ), + sampler ); + } +}