From 22d97b2f5889a3a9fa64a633bd3acfcbb908bd9b Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Tue, 23 May 2017 10:40:10 +0200 Subject: [PATCH] IndexPopulator for native schema number index Implemented using GBPTree and for the time being only IndexPopulator is implemented, online IndexAccessor and SchemaIndexProvider will come later. Both unique and non-unique layouts are supported and all types of Numbers. There are small differences between unique and non-unique layouts where entity ID is in the key for non-unique, but in the value for unique. This makes the keys smaller for the unique layout, something which will make the overall index size smaller due to internal tree nodes not containing values. Each layout contains both the comparison value, a coerced value type which all numbers are converted into for comparison, but also their original value. This means that there will not need to be an additional entity property lookup for filtering those results higher up in the stack. --- .../src/test/java/org/neo4j/test/Randoms.java | 37 +- .../java/org/neo4j/test/rule/RandomRule.java | 5 + .../index/internal/gbptree/CleanupJob.java | 3 + .../neo4j/index/internal/gbptree/GBPTree.java | 14 +- .../GroupingRecoveryCleanupWorkCollector.java | 3 + .../neo4j/index/internal/gbptree/Header.java | 1 - .../gbptree/RecoveryCleanupWorkCollector.java | 4 + .../index/sampling/NonUniqueIndexSampler.java | 23 + .../neo4j/kernel/impl/index/GBPTreeUtil.java | 92 ++ .../index/labelscan/NativeLabelScanStore.java | 25 +- .../schema/ConflictDetectingValueMerger.java | 70 ++ .../index/schema/FailureHeaderWriter.java | 53 ++ .../schema/FullScanNonUniqueIndexSampler.java | 82 ++ .../schema/NativeSchemaIndexPopulator.java | 223 +++++ .../NonUniqueNativeSchemaIndexPopulator.java | 109 +++ .../NonUniqueSchemaNumberIndexLayout.java | 117 +++ .../schema/NonUniqueSchemaNumberKey.java | 72 ++ .../schema/NonUniqueSchemaNumberValue.java | 54 ++ .../impl/index/schema/SamplingUtil.java | 43 + .../impl/index/schema/SchemaNumberKey.java | 34 + .../impl/index/schema/SchemaNumberValue.java | 69 ++ .../schema/SchemaNumberValueConversion.java | 63 ++ .../UniqueNativeSchemaIndexPopulator.java | 63 ++ .../schema/UniqueSchemaNumberIndexLayout.java | 129 +++ .../index/schema/UniqueSchemaNumberKey.java | 70 ++ .../index/schema/UniqueSchemaNumberValue.java | 59 ++ .../FullScanNonUniqueIndexSamplerTest.java | 140 +++ .../NativeSchemaIndexPopulatorTest.java | 849 ++++++++++++++++++ ...nUniqueNativeSchemaIndexPopulatorTest.java | 216 +++++ .../UniqueNativeSchemaIndexPopulatorTest.java | 190 ++++ .../DirectNonUniqueIndexSampler.java | 27 +- 31 files changed, 2883 insertions(+), 56 deletions(-) create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/GBPTreeUtil.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/ConflictDetectingValueMerger.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FailureHeaderWriter.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FullScanNonUniqueIndexSampler.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexPopulator.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueNativeSchemaIndexPopulator.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberIndexLayout.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberKey.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberValue.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SamplingUtil.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberKey.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberValue.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberValueConversion.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueNativeSchemaIndexPopulator.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberIndexLayout.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberKey.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberValue.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/FullScanNonUniqueIndexSamplerTest.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexPopulatorTest.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NonUniqueNativeSchemaIndexPopulatorTest.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/UniqueNativeSchemaIndexPopulatorTest.java diff --git a/community/common/src/test/java/org/neo4j/test/Randoms.java b/community/common/src/test/java/org/neo4j/test/Randoms.java index dbde68d053b3f..20f77188e7aee 100644 --- a/community/common/src/test/java/org/neo4j/test/Randoms.java +++ b/community/common/src/test/java/org/neo4j/test/Randoms.java @@ -155,11 +155,16 @@ public char character( int characterSets ) { switch ( bit ) { - case CS_LOWERCASE_LETTERS: return (char) intBetween( 'a', 'z' ); - case CS_UPPERCASE_LETTERS: return (char) intBetween( 'A', 'Z' ); - case CS_DIGITS: return (char) intBetween( '0', '9' ); - case CS_SYMBOLS: return symbol(); - default: throw new IllegalArgumentException( "Unknown character set " + bit ); + case CS_LOWERCASE_LETTERS: + return (char) intBetween( 'a', 'z' ); + case CS_UPPERCASE_LETTERS: + return (char) intBetween( 'A', 'Z' ); + case CS_DIGITS: + return (char) intBetween( '0', '9' ); + case CS_SYMBOLS: + return symbol(); + default: + throw new IllegalArgumentException( "Unknown character set " + bit ); } } } @@ -211,6 +216,28 @@ public T among( T[] among ) return among[random.nextInt( among.length )]; } + public Number numberPropertyValue() + { + int type = random.nextInt( 6 ); + switch ( type ) + { + case 0: + return (byte) random.nextInt(); + case 1: + return (short) random.nextInt(); + case 2: + return random.nextInt(); + case 3: + return random.nextLong(); + case 4: + return random.nextFloat(); + case 5: + return random.nextDouble(); + default: + throw new IllegalArgumentException( "Unknown value type " + type ); + } + } + public Object propertyValue() { return propertyValue( propertyType( true ) ); diff --git a/community/common/src/test/java/org/neo4j/test/rule/RandomRule.java b/community/common/src/test/java/org/neo4j/test/rule/RandomRule.java index 3893170b99700..36638d1617008 100644 --- a/community/common/src/test/java/org/neo4j/test/rule/RandomRule.java +++ b/community/common/src/test/java/org/neo4j/test/rule/RandomRule.java @@ -177,6 +177,11 @@ public T among( T[] among ) return randoms.among( among ); } + public Number numberPropertyValue() + { + return randoms.numberPropertyValue(); + } + public Object propertyValue() { return randoms.propertyValue(); diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/CleanupJob.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/CleanupJob.java index 51566867910bc..80839dc321f79 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/CleanupJob.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/CleanupJob.java @@ -42,6 +42,9 @@ public interface CleanupJob extends Runnable */ Exception getCause(); + /** + * A {@link CleanupJob} that doesn't need cleaning, i.e. it's already clean. + */ CleanupJob CLEAN = new CleanupJob() { @Override diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java index 702dab3903251..dc3a4bc160f7a 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java @@ -203,7 +203,7 @@ public void startupState( boolean clean ) /** * No-op header reader. */ - static final Header.Reader NO_HEADER = ( cursor, length ) -> + public static final Header.Reader NO_HEADER = ( cursor, length ) -> { }; @@ -409,7 +409,7 @@ public GBPTree( PageCache pageCache, File indexFile, Layout layout, i { close(); } - catch ( IOException e ) + catch ( Throwable e ) { t.addSuppressed( e ); } @@ -946,7 +946,15 @@ void printTree() throws IOException } // Utility method - void printTree( boolean printValues, boolean printPosition, boolean printState ) throws IOException + /** + * Prints the contents of the tree to System.out. + * + * @param printValues whether or not to print values in the leaf nodes. + * @param printPosition whether or not to print position for each key. + * @param printState whether or not to print the tree state. + * @throws IOException on I/O error. + */ + public void printTree( boolean printValues, boolean printPosition, boolean printState ) throws IOException { try ( PageCursor cursor = openRootCursor( PagedFile.PF_SHARED_READ_LOCK ) ) { diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/GroupingRecoveryCleanupWorkCollector.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/GroupingRecoveryCleanupWorkCollector.java index b34ef50bd27be..0d78e1422adfc 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/GroupingRecoveryCleanupWorkCollector.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/GroupingRecoveryCleanupWorkCollector.java @@ -36,6 +36,9 @@ public class GroupingRecoveryCleanupWorkCollector implements RecoveryCleanupWork private final Queue jobs; private final JobScheduler jobScheduler; + /** + * @param jobScheduler {@link JobScheduler} to queue {@link CleanupJob} into. + */ public GroupingRecoveryCleanupWorkCollector( JobScheduler jobScheduler ) { this.jobScheduler = jobScheduler; diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/Header.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/Header.java index 864105b5c1729..1b9e53260e146 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/Header.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/Header.java @@ -38,7 +38,6 @@ public interface Writer /** * Writes header data into {@code to} with previous valid header data found in {@code from} of {@code length} * bytes in size. - * * @param from {@link PageCursor} positioned at the header data written in the previous check point. * @param length size in bytes of the previous header data. * @param to {@link PageCursor} to write new header into. diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/RecoveryCleanupWorkCollector.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/RecoveryCleanupWorkCollector.java index 771d459e3f011..6aca12764abf7 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/RecoveryCleanupWorkCollector.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/RecoveryCleanupWorkCollector.java @@ -45,6 +45,10 @@ public interface RecoveryCleanupWorkCollector extends Lifecycle */ RecoveryCleanupWorkCollector IMMEDIATE = new ImmediateRecoveryCleanupWorkCollector(); + /** + * {@link RecoveryCleanupWorkCollector} which runs added {@link CleanupJob} as part of the {@link #add(CleanupJob)} + * call in the caller thread. + */ class ImmediateRecoveryCleanupWorkCollector extends LifecycleAdapter implements RecoveryCleanupWorkCollector { @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/sampling/NonUniqueIndexSampler.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/sampling/NonUniqueIndexSampler.java index 4f88183a4ca17..756e2f00a976f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/sampling/NonUniqueIndexSampler.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/sampling/NonUniqueIndexSampler.java @@ -40,4 +40,27 @@ public interface NonUniqueIndexSampler IndexSample result(); IndexSample result( int numDocs ); + + abstract class Adapter implements NonUniqueIndexSampler + { + @Override + public void include( String value ) + { // no-op + } + + @Override + public void include( String value, long increment ) + { // no-op + } + + @Override + public void exclude( String value ) + { // no-op + } + + @Override + public void exclude( String value, long decrement ) + { // no-op + } + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/GBPTreeUtil.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/GBPTreeUtil.java new file mode 100644 index 0000000000000..478fb2e9fb8e8 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/GBPTreeUtil.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index; + +import java.io.File; +import java.io.IOException; +import java.nio.file.NoSuchFileException; + +import org.neo4j.index.internal.gbptree.GBPTree; +import org.neo4j.io.fs.FileHandle; +import org.neo4j.io.pagecache.PageCache; + +/** + * Utilities for common operations around a {@link GBPTree}. + */ +public class GBPTreeUtil +{ + /** + * Deletes store file backing a {@link GBPTree}. + * + * @param pageCache {@link PageCache} which manages the file. + * @param storeFile the {@link File} to delete. + * @throws NoSuchFileException if the {@code storeFile} doesn't exist according to the {@code pageCache}. + * @throws IOException on failure to delete existing {@code storeFile}. + */ + public static void delete( PageCache pageCache, File storeFile ) throws IOException + { + FileHandle fileHandle = storeFileHandle( pageCache, storeFile ); + fileHandle.delete(); + } + + /** + * Deletes store file backing a {@link GBPTree}, if it exists according to the {@code pageCache}. + * + * @param pageCache {@link PageCache} which manages the file. + * @param storeFile the {@link File} to delete. + * @throws IOException on failure to delete existing {@code storeFile}. + */ + public static void deleteIfPresent( PageCache pageCache, File storeFile ) throws IOException + { + try + { + delete( pageCache, storeFile ); + } + catch ( NoSuchFileException e ) + { + // File does not exist, we don't need to delete + } + } + + /** + * Checks whether or not {@code storeFile} exists according to {@code pageCache}. + * + * @param pageCache {@link PageCache} which manages the file. + * @param storeFile the {@link File} to check for existence. + * @return {@code true} if {@code storeFile} exists according to {@code pageCache}, otherwise {@code false}. + */ + public static boolean storeFileExists( PageCache pageCache, File storeFile ) + { + try + { + storeFileHandle( pageCache, storeFile ); + return true; + } + catch ( IOException e ) + { + return false; + } + } + + private static FileHandle storeFileHandle( PageCache pageCache, File storeFile ) throws IOException + { + return pageCache.getCachedFileSystem().streamFilesRecursive( storeFile ).findFirst().get(); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java index bdc00937b20a8..92638dff7b500 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeLabelScanStore.java @@ -26,19 +26,17 @@ import java.io.UncheckedIOException; import java.nio.file.NoSuchFileException; import java.util.function.Consumer; -import java.util.Optional; import java.util.function.IntFunction; import org.neo4j.cursor.RawCursor; import org.neo4j.graphdb.ResourceIterator; import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.index.internal.gbptree.GBPTree; import org.neo4j.index.internal.gbptree.Header; import org.neo4j.index.internal.gbptree.Hit; import org.neo4j.index.internal.gbptree.Layout; import org.neo4j.index.internal.gbptree.MetadataMismatchException; -import org.neo4j.io.fs.FileHandle; +import aorg.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; @@ -46,6 +44,7 @@ import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.api.labelscan.LabelScanWriter; import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream; +import org.neo4j.kernel.impl.index.GBPTreeUtil; import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.storageengine.api.schema.LabelScanReader; @@ -337,19 +336,7 @@ public void init() throws IOException @Override public boolean hasStore() throws IOException { - try - { - return storeFileHandle().isPresent(); - } - catch ( NoSuchFileException e ) - { - return false; - } - } - - private Optional storeFileHandle() throws IOException - { - return pageCache.getCachedFileSystem().streamFilesRecursive( storeFile ).findFirst() ; + return GBPTreeUtil.storeFileExists( pageCache, storeFile ); } /** @@ -403,11 +390,7 @@ private void dropStrict() throws IOException index.close(); index = null; } - Optional fileHandle = storeFileHandle(); - if ( fileHandle.isPresent() ) - { - fileHandle.get().delete(); - } + GBPTreeUtil.delete( pageCache, storeFile ); } /** diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/ConflictDetectingValueMerger.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/ConflictDetectingValueMerger.java new file mode 100644 index 0000000000000..723eaf53d1b34 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/ConflictDetectingValueMerger.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import org.neo4j.index.internal.gbptree.ValueMerger; +import org.neo4j.index.internal.gbptree.Writer; + +/** + * {@link ValueMerger} which will merely detect conflict, not change any value if conflict, i.e. if the + * key already exists. After this merge has been used in a call to {@link Writer#merge(Object, Object, ValueMerger)} + * the {@link #wasConflict()} accessor can be called to check whether or not that call conflicted with + * an existing key. A call to {@link #wasConflict()} will also clear the conflict flag. + * + * @param type of values being merged. + */ +class ConflictDetectingValueMerger implements ValueMerger +{ + private boolean conflict; + private long existingNodeId; + private long addedNodeId; + + @Override + public VALUE merge( VALUE existingValue, VALUE newValue ) + { + conflict = true; + existingNodeId = existingValue.getEntityId(); + addedNodeId = newValue.getEntityId(); + return null; + } + + /** + * @return whether or not merge conflicted with an existing key. This call also clears the conflict flag. + */ + boolean wasConflict() + { + boolean result = conflict; + if ( conflict ) + { + conflict = false; + } + return result; + } + + long existingNodeId() + { + return existingNodeId; + } + + long addedNodeId() + { + return addedNodeId; + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FailureHeaderWriter.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FailureHeaderWriter.java new file mode 100644 index 0000000000000..6cf02ccdbe0a2 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FailureHeaderWriter.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import java.util.Arrays; +import java.util.function.Consumer; + +import org.neo4j.index.internal.gbptree.GBPTree; +import org.neo4j.io.pagecache.PageCursor; + +/** + * Writes a failure message to a header in a {@link GBPTree}. + */ +class FailureHeaderWriter implements Consumer +{ + private final byte[] failureBytes; + + FailureHeaderWriter( byte[] failureBytes ) + { + this.failureBytes = failureBytes; + } + + @Override + public void accept( PageCursor cursor ) + { + byte[] bytesToWrite = failureBytes; + cursor.putByte( NativeSchemaIndexPopulator.BYTE_FAILED ); + int availableSpace = cursor.getCurrentPageSize() - cursor.getOffset(); + if ( bytesToWrite.length + 2 > availableSpace ) + { + bytesToWrite = Arrays.copyOf( bytesToWrite, availableSpace - 2 ); + } + cursor.putShort( (short) bytesToWrite.length ); + cursor.putBytes( bytesToWrite ); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FullScanNonUniqueIndexSampler.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FullScanNonUniqueIndexSampler.java new file mode 100644 index 0000000000000..6a6ae4932ae98 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FullScanNonUniqueIndexSampler.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import java.io.IOException; + +import org.neo4j.cursor.RawCursor; +import org.neo4j.index.internal.gbptree.GBPTree; +import org.neo4j.index.internal.gbptree.Hit; +import org.neo4j.index.internal.gbptree.Layout; +import org.neo4j.kernel.impl.api.index.sampling.DefaultNonUniqueIndexSampler; +import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; +import org.neo4j.kernel.impl.api.index.sampling.NonUniqueIndexSampler; +import org.neo4j.storageengine.api.schema.IndexSample; + +/** + * {@link NonUniqueIndexSampler} which performs a full scans of a {@link GBPTree} in {@link #result()}. + * + * @param type of keys in tree. + * @param type of values in tree. + */ +class FullScanNonUniqueIndexSampler + extends NonUniqueIndexSampler.Adapter +{ + private final GBPTree gbpTree; + private final Layout layout; + private final IndexSamplingConfig samplingConfig; + + FullScanNonUniqueIndexSampler( GBPTree gbpTree, Layout layout, + IndexSamplingConfig samplingConfig ) + { + this.gbpTree = gbpTree; + this.layout = layout; + this.samplingConfig = samplingConfig; + } + + @Override + public IndexSample result() + { + KEY lowest = layout.newKey(); + lowest.initAsLowest(); + KEY highest = layout.newKey(); + highest.initAsHighest(); + try ( RawCursor,IOException> seek = gbpTree.seek( lowest, highest ) ) + { + NonUniqueIndexSampler sampler = new DefaultNonUniqueIndexSampler( samplingConfig.sampleSizeLimit() ); + while ( seek.next() ) + { + Hit hit = seek.get(); + sampler.include( hit.key().propertiesAsString() ); + } + return sampler.result(); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + } + + @Override + public IndexSample result( int numDocs ) + { + throw new UnsupportedOperationException(); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexPopulator.java new file mode 100644 index 0000000000000..ee132a8a7b9bb --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexPopulator.java @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collection; + +import org.neo4j.collection.primitive.PrimitiveLongSet; +import org.neo4j.index.internal.gbptree.GBPTree; +import org.neo4j.index.internal.gbptree.Layout; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.index.internal.gbptree.Writer; +import org.neo4j.io.pagecache.IOLimiter; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; +import org.neo4j.kernel.api.index.IndexEntryUpdate; +import org.neo4j.kernel.api.index.IndexPopulator; +import org.neo4j.kernel.api.index.IndexUpdater; +import org.neo4j.kernel.api.index.PropertyAccessor; +import org.neo4j.kernel.impl.index.GBPTreeUtil; + +import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER; +import static org.neo4j.index.internal.gbptree.GBPTree.NO_MONITOR; +import static org.neo4j.kernel.api.schema.OrderedPropertyValues.ofUndefined; + +/** + * {@link IndexPopulator} backed by a {@link GBPTree}. + * + * @param type of {@link SchemaNumberKey}. + * @param type of {@link SchemaNumberValue}. + */ +public abstract class NativeSchemaIndexPopulator + implements IndexPopulator +{ + static final byte BYTE_ONLINE = 1; + static final byte BYTE_FAILED = 0; + + private final PageCache pageCache; + private final File storeFile; + private final KEY treeKey; + private final VALUE treeValue; + private final RecoveryCleanupWorkCollector recoveryCleanupWorkCollector; + private final ConflictDetectingValueMerger conflictDetectingValueMerger; + protected final Layout layout; + + private Writer singleWriter; + private byte[] failureBytes; + + GBPTree gbpTree; + + NativeSchemaIndexPopulator( PageCache pageCache, File storeFile, Layout layout, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector ) + { + this.pageCache = pageCache; + this.storeFile = storeFile; + this.layout = layout; + this.treeKey = layout.newKey(); + this.treeValue = layout.newValue(); + this.recoveryCleanupWorkCollector = recoveryCleanupWorkCollector; + this.conflictDetectingValueMerger = new ConflictDetectingValueMerger<>(); + } + + @Override + public void create() throws IOException + { + GBPTreeUtil.deleteIfPresent( pageCache, storeFile ); + gbpTree = new GBPTree<>( pageCache, storeFile, layout, 0, NO_MONITOR, NO_HEADER, + recoveryCleanupWorkCollector ); + instantiateWriter(); + } + + protected void instantiateWriter() throws IOException + { + assert singleWriter == null; + singleWriter = gbpTree.writer(); + } + + @Override + public void drop() throws IOException + { + closeWriter(); + gbpTree = closeIfPresent( gbpTree ); + GBPTreeUtil.deleteIfPresent( pageCache, storeFile ); + } + + @Override + public void add( Collection> updates ) throws IndexEntryConflictException, IOException + { + for ( IndexEntryUpdate update : updates ) + { + add( update ); + } + } + + @Override + public void add( IndexEntryUpdate update ) throws IndexEntryConflictException, IOException + { + treeKey.from( update.getEntityId(), update.values() ); + treeValue.from( update.getEntityId(), update.values() ); + singleWriter.merge( treeKey, treeValue, conflictDetectingValueMerger ); + if ( conflictDetectingValueMerger.wasConflict() ) + { + long existingNodeId = conflictDetectingValueMerger.existingNodeId(); + long addedNodeId = conflictDetectingValueMerger.addedNodeId(); + // TODO: not sure about the OrderedPropertyValues#ofUndefined bit + throw new IndexEntryConflictException( existingNodeId, addedNodeId, ofUndefined( update.values() ) ); + } + } + + @Override + public void verifyDeferredConstraints( PropertyAccessor propertyAccessor ) + throws IndexEntryConflictException, IOException + { + // No-op, uniqueness is checked for each update in add(IndexEntryUpdate) + } + + @Override + public IndexUpdater newPopulatingUpdater( PropertyAccessor accessor ) throws IOException + { + return new NativeSchemaIndexUpdater(); + } + + @Override + public void close( boolean populationCompletedSuccessfully ) throws IOException + { + if ( populationCompletedSuccessfully && failureBytes != null ) + { + throw new IllegalStateException( "Can't mark index as online after it has been marked as failure" ); + } + closeWriter(); + if ( populationCompletedSuccessfully ) + { + markTreeAsOnline(); + } + else + { + markTreeAsFailed(); + } + gbpTree.close(); + gbpTree = null; +} + + @Override + public void markAsFailed( String failure ) throws IOException + { + failureBytes = failure.getBytes( StandardCharsets.UTF_8 ); + } + + private void markTreeAsFailed() throws IOException + { + if ( failureBytes == null ) + { + failureBytes = "".getBytes(); + } + gbpTree.checkpoint( IOLimiter.unlimited(), new FailureHeaderWriter( failureBytes ) ); + } + + private void markTreeAsOnline() throws IOException + { + gbpTree.checkpoint( IOLimiter.unlimited(), ( pc ) -> pc.putByte( BYTE_ONLINE ) ); + } + + private T closeIfPresent( T closeable ) throws IOException + { + if ( closeable != null ) + { + closeable.close(); + } + return null; + } + + protected void closeWriter() throws IOException + { + singleWriter = closeIfPresent( singleWriter ); + } + + private class NativeSchemaIndexUpdater implements IndexUpdater + { + private boolean closed; + + @Override + public void process( IndexEntryUpdate update ) throws IOException, IndexEntryConflictException + { + if ( closed ) + { + throw new IllegalStateException( "Index updater has been closed." ); + } + add( update ); + } + + @Override + public void remove( PrimitiveLongSet nodeIds ) throws IOException + { + throw new UnsupportedOperationException( "Implement me" ); + } + + @Override + public void close() throws IOException, IndexEntryConflictException + { + closed = true; + } + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueNativeSchemaIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueNativeSchemaIndexPopulator.java new file mode 100644 index 0000000000000..583946e40cbed --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueNativeSchemaIndexPopulator.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import java.io.File; +import java.io.IOException; + +import org.neo4j.index.internal.gbptree.Layout; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.api.index.IndexEntryUpdate; +import org.neo4j.kernel.impl.api.index.sampling.DefaultNonUniqueIndexSampler; +import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; +import org.neo4j.kernel.impl.api.index.sampling.NonUniqueIndexSampler; +import org.neo4j.storageengine.api.schema.IndexSample; + +/** + * {@link NativeSchemaIndexPopulator} which can accept duplicate values (for different entity ids). + */ +class NonUniqueNativeSchemaIndexPopulator + extends NativeSchemaIndexPopulator +{ + private final IndexSamplingConfig samplingConfig; + private boolean updateSampling; + private NonUniqueIndexSampler sampler; + + NonUniqueNativeSchemaIndexPopulator( PageCache pageCache, File storeFile, Layout layout, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, IndexSamplingConfig samplingConfig ) + { + super( pageCache, storeFile, layout, recoveryCleanupWorkCollector ); + this.samplingConfig = samplingConfig; + } + + @Override + public void includeSample( IndexEntryUpdate update ) + { + if ( updateSampling ) + { + checkSampler(); + sampler.include( SamplingUtil.encodedStringValuesForSampling( update.values() ) ); + } + } + + @Override + public void configureSampling( boolean onlineSampling ) + { + this.updateSampling = onlineSampling; + this.sampler = onlineSampling ? new DefaultNonUniqueIndexSampler( samplingConfig.sampleSizeLimit() ) + : new FullScanNonUniqueIndexSampler<>( gbpTree, layout, samplingConfig ); + } + + @Override + public IndexSample sampleResult() + { + checkSampler(); + + // Close the writer before scanning + try + { + closeWriter(); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + + try + { + return sampler.result(); + } + finally + { + // Start the writer again (TODO may be unnecessary) + try + { + instantiateWriter(); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + } + } + + private void checkSampler() + { + if ( sampler == null ) + { + throw new IllegalStateException( "Please configure populator sampler before using it." ); + } + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberIndexLayout.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberIndexLayout.java new file mode 100644 index 0000000000000..ae1462961bb8b --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberIndexLayout.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import org.neo4j.index.internal.gbptree.Layout; +import org.neo4j.io.pagecache.PageCursor; + +/** + * {@link Layout} for numbers where numbers doesn't need to be unique. + */ +public class NonUniqueSchemaNumberIndexLayout implements Layout +{ + private static final String IDENTIFIER_NAME = "NUNI"; + + @Override + public NonUniqueSchemaNumberKey newKey() + { + return new NonUniqueSchemaNumberKey(); + } + + @Override + public NonUniqueSchemaNumberKey copyKey( NonUniqueSchemaNumberKey key, + NonUniqueSchemaNumberKey into ) + { + into.value = key.value; + into.entityId = key.entityId; + return into; + } + + @Override + public NonUniqueSchemaNumberValue newValue() + { + return new NonUniqueSchemaNumberValue(); + } + + @Override + public int keySize() + { + return NonUniqueSchemaNumberKey.SIZE; + } + + @Override + public int valueSize() + { + return NonUniqueSchemaNumberValue.SIZE; + } + + @Override + public void writeKey( PageCursor cursor, NonUniqueSchemaNumberKey key ) + { + cursor.putLong( Double.doubleToRawLongBits( key.value ) ); + cursor.putLong( key.entityId ); + } + + @Override + public void writeValue( PageCursor cursor, NonUniqueSchemaNumberValue key ) + { + cursor.putByte( key.type ); + cursor.putLong( key.rawValueBits ); + } + + @Override + public void readKey( PageCursor cursor, NonUniqueSchemaNumberKey into ) + { + into.value = Double.longBitsToDouble( cursor.getLong() ); + into.entityId = cursor.getLong(); + } + + @Override + public void readValue( PageCursor cursor, NonUniqueSchemaNumberValue into ) + { + into.type = cursor.getByte(); + into.rawValueBits = cursor.getLong(); + } + + @Override + public long identifier() + { + return Layout.namedIdentifier( IDENTIFIER_NAME, NonUniqueSchemaNumberValue.SIZE ); + } + + @Override + public int majorVersion() + { + return 0; + } + + @Override + public int minorVersion() + { + return 1; + } + + @Override + public int compare( NonUniqueSchemaNumberKey o1, NonUniqueSchemaNumberKey o2 ) + { + int compare = Double.compare( o1.value, o2.value ); + return compare != 0 ? compare : Long.compare( o1.entityId, o2.entityId ); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberKey.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberKey.java new file mode 100644 index 0000000000000..8cb05798e41b2 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberKey.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValueConversion.assertValidSingleNumberPropertyValue; + +/** + * Includes comparison value and entity id (to be able to handle non-unique values). + * Comparison value is basically any number as a double, a conversion which is lossy by nature, + * especially for higher decimal values. Actual value is stored in {@link SchemaNumberValue} + * for ability to filter accidental coersions directly internally. + */ +class NonUniqueSchemaNumberKey implements SchemaNumberKey +{ + static final int SIZE = + Long.SIZE + /* compare value (double represented by long) */ + Long.SIZE; /* entityId */ + + double value; + long entityId; + + @Override + public void from( long entityId, Object[] values ) + { + assertValidSingleNumberPropertyValue( values ); + this.value = ((Number) values[0]).doubleValue(); + this.entityId = entityId; + } + + @Override + public String propertiesAsString() + { + return String.valueOf( value ); + } + + @Override + public void initAsLowest() + { + value = -Double.MAX_VALUE; + entityId = Long.MIN_VALUE; + } + + @Override + public void initAsHighest() + { + value = Double.MAX_VALUE; + entityId = Long.MAX_VALUE; + } + + @Override + public String toString() + { + return "compareValue=" + value + ",entityId=" + entityId; + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberValue.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberValue.java new file mode 100644 index 0000000000000..29c3f54af64ba --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NonUniqueSchemaNumberValue.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import static java.lang.String.format; + +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValueConversion.assertValidSingleNumberPropertyValue; +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValueConversion.toValue; + +/** + * Relies on its key counterpart to supply entity id, since the key needs entity id anyway. + */ +class NonUniqueSchemaNumberValue extends SchemaNumberValue +{ + static final int SIZE = + Byte.SIZE + /* type */ + Long.SIZE; /* value bits */ + + @Override + public void from( long entityId, Object[] values ) + { + assertValidSingleNumberPropertyValue( values ); + extractValue( (Number) values[0] ); + } + + @Override + public long getEntityId() + { + throw new UnsupportedOperationException( "entity id should be retrieved from key for non-unique index" ); + } + + @Override + public String toString() + { + return format( "type=%d,value=%s", type, toValue( type, rawValueBits ) ); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SamplingUtil.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SamplingUtil.java new file mode 100644 index 0000000000000..646bcb4f4c118 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SamplingUtil.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import org.neo4j.storageengine.api.schema.IndexSampler; + +/** + * Utilities for implementing {@link IndexSampler sampling}. + */ +class SamplingUtil +{ + private static final String DELIMITER = "\u001F"; + + static String encodedStringValuesForSampling( Object... values ) + { + StringBuilder sb = new StringBuilder(); + String sep = ""; + for ( Object value : values ) + { + sb.append( sep ); + sep = DELIMITER; + sb.append( value.toString() ); + } + return sb.toString(); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberKey.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberKey.java new file mode 100644 index 0000000000000..7c6d47e1d9b70 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberKey.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +/** + * Key in a native schema index for numbers. + */ +interface SchemaNumberKey +{ + void from( long entityId, Object[] values ); + + String propertiesAsString(); + + void initAsLowest(); + + void initAsHighest(); +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberValue.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberValue.java new file mode 100644 index 0000000000000..31d6b915a0761 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberValue.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import org.neo4j.index.internal.gbptree.GBPTree; + +/** + * Value in a {@link GBPTree} handling numbers suitable for schema indexing. + * Contains actual number for internal filtering after accidental query hits due to double value coersion. + */ +abstract class SchemaNumberValue +{ + static final byte LONG = 0; + static final byte FLOAT = 1; + static final byte DOUBLE = 2; + + protected byte type; + protected long rawValueBits; + + abstract void from( long entityId, Object[] values ); + + abstract long getEntityId(); + + byte type() + { + return type; + } + + long rawValueBits() + { + return rawValueBits; + } + + void extractValue( Number value ) + { + if ( value instanceof Double ) + { + type = DOUBLE; + rawValueBits = Double.doubleToLongBits( (Double) value ); + } + else if ( value instanceof Float ) + { + type = FLOAT; + rawValueBits = Float.floatToIntBits( (Float) value ); + } + else + { + type = LONG; + rawValueBits = value.longValue(); + } + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberValueConversion.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberValueConversion.java new file mode 100644 index 0000000000000..b4e125232a56d --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SchemaNumberValueConversion.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValue.DOUBLE; +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValue.FLOAT; +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValue.LONG; + +/** + * Utilities for converting number values to and from different representations. + */ +class SchemaNumberValueConversion +{ + static void assertValidSingleNumberPropertyValue( Object[] values ) + { + // TODO: support multiple values, right? + if ( values.length > 1 ) + { + throw new IllegalArgumentException( "Tried to create composite key with non-composite schema key layout" ); + } + if ( values.length < 1 ) + { + throw new IllegalArgumentException( "Tried to create key without value" ); + } + if ( !(values[0] instanceof Number) ) + { + throw new IllegalArgumentException( + "Key layout does only support numbers, tried to create key from " + values[0] ); + } + } + + static Number toValue( byte type, long rawValueBits ) + { + switch ( type ) + { + case LONG: + return rawValueBits; + case FLOAT: + return Float.intBitsToFloat( (int)rawValueBits ); + case DOUBLE: + return Double.longBitsToDouble( rawValueBits ); + default: + throw new IllegalArgumentException( "Unexpected type " + type ); + } + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueNativeSchemaIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueNativeSchemaIndexPopulator.java new file mode 100644 index 0000000000000..d6409cfc66aed --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueNativeSchemaIndexPopulator.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import java.io.File; + +import org.neo4j.index.internal.gbptree.Layout; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.api.index.IndexEntryUpdate; +import org.neo4j.kernel.impl.api.index.sampling.UniqueIndexSampler; +import org.neo4j.storageengine.api.schema.IndexSample; + +/** + * {@link NativeSchemaIndexPopulator} which can enforces unique values. + */ +class UniqueNativeSchemaIndexPopulator + extends NativeSchemaIndexPopulator +{ + private final UniqueIndexSampler sampler; + + UniqueNativeSchemaIndexPopulator( PageCache pageCache, File storeFile, Layout layout, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector ) + { + super( pageCache, storeFile, layout, recoveryCleanupWorkCollector ); + this.sampler = new UniqueIndexSampler(); + } + + @Override + public void includeSample( IndexEntryUpdate update ) + { + sampler.increment( 1 ); + } + + @Override + public void configureSampling( boolean onlineSampling ) + { + // nothing to configure so far + } + + @Override + public IndexSample sampleResult() + { + return sampler.result(); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberIndexLayout.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberIndexLayout.java new file mode 100644 index 0000000000000..78ef840a05fca --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberIndexLayout.java @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import org.neo4j.index.internal.gbptree.Layout; +import org.neo4j.io.pagecache.PageCursor; + +/** + * {@link Layout} for numbers where numbers need to be unique. + */ +class UniqueSchemaNumberIndexLayout implements Layout +{ + private static final String IDENTIFIER_NAME = "UNI"; + + @Override + public UniqueSchemaNumberKey newKey() + { + return new UniqueSchemaNumberKey(); + } + + @Override + public UniqueSchemaNumberKey copyKey( UniqueSchemaNumberKey key, UniqueSchemaNumberKey into ) + { + into.value = key.value; + into.isHighest = key.isHighest; + return into; + } + + @Override + public UniqueSchemaNumberValue newValue() + { + return new UniqueSchemaNumberValue(); + } + + @Override + public int keySize() + { + return UniqueSchemaNumberKey.SIZE; + } + + @Override + public int valueSize() + { + return UniqueSchemaNumberValue.SIZE; + } + + @Override + public void writeKey( PageCursor cursor, UniqueSchemaNumberKey key ) + { + cursor.putLong( Double.doubleToRawLongBits( key.value ) ); + } + + @Override + public void readKey( PageCursor cursor, UniqueSchemaNumberKey into ) + { + into.value = Double.longBitsToDouble( cursor.getLong() ); + } + + @Override + public void writeValue( PageCursor cursor, UniqueSchemaNumberValue value ) + { + cursor.putByte( value.type ); + cursor.putLong( value.rawValueBits ); + cursor.putLong( value.entityId ); + } + + @Override + public void readValue( PageCursor cursor, UniqueSchemaNumberValue into ) + { + into.type = cursor.getByte(); + into.rawValueBits = cursor.getLong(); + into.entityId = cursor.getLong(); + } + + @Override + public long identifier() + { + return Layout.namedIdentifier( IDENTIFIER_NAME, UniqueSchemaNumberValue.SIZE ); + } + + @Override + public int majorVersion() + { + return 0; + } + + @Override + public int minorVersion() + { + return 1; + } + + @Override + public int compare( UniqueSchemaNumberKey o1, UniqueSchemaNumberKey o2 ) + { + int comparison = Double.compare( o1.value, o2.value ); + if ( comparison == 0 ) + { + // This is a special case where we need to check if o2 is the unbounded max key. + // This needs to be checked to be able to support storing Double.MAX_VALUE and retrieve it later. + if ( o2.isHighest ) + { + return -1; + } + if ( o1.isHighest ) + { + return 1; + } + } + return comparison; + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberKey.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberKey.java new file mode 100644 index 0000000000000..19ab1c5027764 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberKey.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValueConversion.assertValidSingleNumberPropertyValue; + +/** + * Contains only comparison value, which means that all values needs to be unique. + * Comparison value is basically any number as a double, a conversion which is lossy by nature, + * especially for higher decimal values. Actual value is stored in {@link SchemaNumberValue} + * for ability to filter accidental coersions directly internally. + */ +class UniqueSchemaNumberKey implements SchemaNumberKey +{ + static final int SIZE = Long.SIZE; + + double value; + boolean isHighest; + + @Override + public void from( long entityId, Object[] values ) + { + assertValidSingleNumberPropertyValue( values ); + value = ((Number) values[0]).doubleValue(); + isHighest = false; + } + + @Override + public String propertiesAsString() + { + throw new UnsupportedOperationException( "Implement me" ); + } + + @Override + public void initAsLowest() + { + value = -Double.MAX_VALUE; + isHighest = false; + } + + @Override + public void initAsHighest() + { + value = Double.MAX_VALUE; + isHighest = true; + } + + @Override + public String toString() + { + return "compareValue=" + value; + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberValue.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberValue.java new file mode 100644 index 0000000000000..6251d723608b1 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/UniqueSchemaNumberValue.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import static java.lang.String.format; + +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValueConversion.assertValidSingleNumberPropertyValue; +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValueConversion.toValue; + +/** + * Adds entity id because its unique key counterpart doesn't have it. The gain of having entity id in value + * is that keys in internal tree nodes becomes smaller so that internal tree nodes can contain more keys. + */ +class UniqueSchemaNumberValue extends SchemaNumberValue +{ + static final int SIZE = + Byte.SIZE + /* type */ + Long.SIZE + /* value bits */ + Long.SIZE; /* entity id TODO 7 bytes could be enough. Also combine with the type byte thing*/ + + long entityId; + + @Override + public void from( long entityId, Object[] values ) + { + assertValidSingleNumberPropertyValue( values ); + extractValue( (Number) values[0] ); + this.entityId = entityId; + } + + @Override + public long getEntityId() + { + return entityId; + } + + @Override + public String toString() + { + return format( "type=%d,value=%s,entityId=%d", type, toValue( type, rawValueBits ), entityId ); + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/FullScanNonUniqueIndexSamplerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/FullScanNonUniqueIndexSamplerTest.java new file mode 100644 index 0000000000000..c73c544f9e8ef --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/FullScanNonUniqueIndexSamplerTest.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.RuleChain; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.neo4j.index.internal.gbptree.GBPTree; +import org.neo4j.index.internal.gbptree.Writer; +import org.neo4j.io.pagecache.IOLimiter; +import org.neo4j.kernel.api.index.IndexEntryUpdate; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; +import org.neo4j.storageengine.api.schema.IndexSample; +import org.neo4j.test.rule.PageCacheRule; +import org.neo4j.test.rule.RandomRule; +import org.neo4j.test.rule.TestDirectory; +import org.neo4j.test.rule.fs.DefaultFileSystemRule; + +import static org.junit.Assert.assertEquals; +import static org.junit.rules.RuleChain.outerRule; + +import static org.neo4j.helpers.collection.Iterators.array; +import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER; +import static org.neo4j.index.internal.gbptree.GBPTree.NO_MONITOR; +import static org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector.IMMEDIATE; +import static org.neo4j.test.rule.PageCacheRule.config; + +public class FullScanNonUniqueIndexSamplerTest +{ + private final DefaultFileSystemRule fs = new DefaultFileSystemRule(); + private final TestDirectory directory = TestDirectory.testDirectory( getClass(), fs.get() ); + private final PageCacheRule pageCacheRule = new PageCacheRule( config().withAccessChecks( true ) ); + protected final RandomRule random = new RandomRule(); + @Rule + public final RuleChain rules = outerRule( fs ).around( directory ).around( pageCacheRule ).around( random ); + + private final NonUniqueSchemaNumberIndexLayout layout = new NonUniqueSchemaNumberIndexLayout(); + + @Test + public void shouldIncludeAllValuesInTree() throws Exception + { + // GIVEN + List values = generateNumberValues(); + buildTree( values ); + + // WHEN + IndexSample sample; + try ( GBPTree gbpTree = newTree( layout ) ) + { + IndexSamplingConfig samplingConfig = new IndexSamplingConfig( Config.defaults() ); + FullScanNonUniqueIndexSampler sampler = + new FullScanNonUniqueIndexSampler<>( gbpTree, layout, samplingConfig ); + sample = sampler.result(); + } + + // THEN + assertEquals( values.size(), sample.sampleSize() ); + assertEquals( countUniqueValues( values ), sample.uniqueValues() ); + assertEquals( values.size(), sample.indexSize() ); + } + + static int countUniqueValues( List values ) + { + int count = 0; + Set seenValues = new HashSet<>(); + for ( Number number : values ) + { + if ( seenValues.add( number.doubleValue() ) ) + { + count++; + } + } + return count; + } + + private List generateNumberValues() + { + List result = new ArrayList<>(); + for ( IndexEntryUpdate update : NativeSchemaIndexPopulatorTest.someDuplicateIndexEntryUpdates() ) + { + result.add( (Number) update.values()[0] ); + } + // TODO: perhaps some more values? + return result; + } + + private void buildTree( List values ) throws IOException + { + try ( GBPTree gbpTree = newTree( layout ); + Writer writer = gbpTree.writer() ) + { + NonUniqueSchemaNumberKey key = layout.newKey(); + NonUniqueSchemaNumberValue value = layout.newValue(); + long nodeId = 0; + for ( Number number : values ) + { + key.from( nodeId, array( number ) ); + value.from( nodeId, array( number ) ); + writer.put( key, value ); + nodeId++; + } + gbpTree.checkpoint( IOLimiter.unlimited() ); + } + } + + private GBPTree + newTree( NonUniqueSchemaNumberIndexLayout layout ) throws IOException + { + return new GBPTree<>( + pageCacheRule.getPageCache( fs ), directory.file( "tree" ), layout, + 0, NO_MONITOR, NO_HEADER, IMMEDIATE ); + } + + // TODO: shouldIncludeHighestAndLowestPossibleNumberValues +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexPopulatorTest.java new file mode 100644 index 0000000000000..78ccf8a5e0719 --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexPopulatorTest.java @@ -0,0 +1,849 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import org.apache.commons.codec.Charsets; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.RuleChain; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Random; +import java.util.Set; + +import org.neo4j.cursor.RawCursor; +import org.neo4j.helpers.collection.PrefetchingIterator; +import org.neo4j.index.internal.gbptree.GBPTree; +import org.neo4j.index.internal.gbptree.Header; +import org.neo4j.index.internal.gbptree.Hit; +import org.neo4j.index.internal.gbptree.Layout; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.io.fs.StoreChannel; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.PageCursor; +import org.neo4j.io.pagecache.PagedFile; +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; +import org.neo4j.kernel.api.index.IndexEntryUpdate; +import org.neo4j.kernel.api.index.IndexUpdater; +import org.neo4j.kernel.api.index.PropertyAccessor; +import org.neo4j.kernel.api.schema.index.IndexDescriptor; +import org.neo4j.kernel.api.schema.index.IndexDescriptorFactory; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; +import org.neo4j.test.rule.PageCacheRule; +import org.neo4j.test.rule.RandomRule; +import org.neo4j.test.rule.TestDirectory; +import org.neo4j.test.rule.fs.DefaultFileSystemRule; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.junit.rules.RuleChain.outerRule; +import static org.neo4j.kernel.impl.index.schema.NativeSchemaIndexPopulator.BYTE_FAILED; +import static org.neo4j.kernel.impl.index.schema.NativeSchemaIndexPopulator.BYTE_ONLINE; +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValue.DOUBLE; +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValue.FLOAT; +import static org.neo4j.kernel.impl.index.schema.SchemaNumberValue.LONG; +import static org.neo4j.test.rule.PageCacheRule.config; + +public abstract class NativeSchemaIndexPopulatorTest +{ + static final int LARGE_AMOUNT_OF_UPDATES = 10_000; + private static final IndexDescriptor indexDescriptor = IndexDescriptorFactory.forLabel( 42, 666 ); + static final PropertyAccessor null_property_accessor = ( nodeId, propKeyId ) -> null; + + private final DefaultFileSystemRule fs = new DefaultFileSystemRule(); + private final TestDirectory directory = TestDirectory.testDirectory( getClass(), fs.get() ); + private final PageCacheRule pageCacheRule = new PageCacheRule( config().withAccessChecks( true ) ); + protected final RandomRule random = new RandomRule(); + @Rule + public final RuleChain rules = outerRule( fs ).around( directory ).around( pageCacheRule ).around( random ); + + private Layout layout; + File indexFile; + PageCache pageCache; + NativeSchemaIndexPopulator populator; + + @Before + public void setup() + { + layout = createLayout(); + indexFile = directory.file( "index" ); + pageCache = pageCacheRule.getPageCache( fs ); + IndexSamplingConfig samplingConfig = new IndexSamplingConfig( Config.embeddedDefaults() ); + populator = createPopulator( pageCache, indexFile, layout, samplingConfig ); + } + + abstract Layout createLayout(); + + abstract NativeSchemaIndexPopulator createPopulator( PageCache pageCache, File indexFile, + Layout layout, IndexSamplingConfig samplingConfig ); + + @Test + public void createShouldCreateFile() throws Exception + { + // given + assertFalse( fs.fileExists( indexFile ) ); + + // when + populator.create(); + + // then + assertTrue( fs.fileExists( indexFile ) ); + populator.close( true ); + } + + @Test + public void createShouldClearExistingFile() throws Exception + { + // given + byte[] someBytes = fileWithContent(); + + // when + populator.create(); + + // then + try ( StoreChannel r = fs.open( indexFile, "r" ) ) + { + byte[] firstBytes = new byte[someBytes.length]; + r.read( ByteBuffer.wrap( firstBytes ) ); + assertNotEquals( "Expected previous file content to have been cleared but was still there", + someBytes, firstBytes ); + } + populator.close( true ); + } + + @Test + public void dropShouldDeleteExistingFile() throws Exception + { + // given + populator.create(); + + // when + populator.drop(); + + // then + assertFalse( fs.fileExists( indexFile ) ); + } + + @Test + public void dropShouldSucceedOnNonExistentFile() throws Exception + { + // given + assertFalse( fs.fileExists( indexFile ) ); + + // then + populator.drop(); + } + + @Test + public void addShouldHandleEmptyCollection() throws Exception + { + // given + populator.create(); + List> updates = Collections.emptyList(); + + // when + populator.add( updates ); + + // then + populator.close( true ); + } + + @Test + public void addShouldApplyAllUpdatesOnce() throws Exception + { + // given + populator.create(); + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] updates = someIndexEntryUpdates(); + + // when + populator.add( Arrays.asList( updates ) ); + + // then + populator.close( true ); + verifyUpdates( updates ); + } + + @Test + public void updaterShouldApplyUpdates() throws Exception + { + // given + populator.create(); + IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ); + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] updates = someIndexEntryUpdates(); + + // when + for ( IndexEntryUpdate update : updates ) + { + updater.process( update ); + } + + // then + populator.close( true ); + verifyUpdates( updates ); + } + + @Test + public void updaterMustThrowIfProcessAfterClose() throws Exception + { + // given + populator.create(); + IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ); + + // when + updater.close(); + + // then + try + { + updater.process( add( 1, Long.MAX_VALUE ) ); + fail( "Expected process to throw on closed updater" ); + } + catch ( IllegalStateException e ) + { + // good + } + populator.close( true ); + } + + @Test + public void shouldApplyInterleavedUpdatesFromAddAndUpdater() throws Exception + { + // given + populator.create(); + IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ); + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] updates = someIndexEntryUpdates(); + + // when + applyInterleaved( updates, updater, populator ); + + // then + populator.close( true ); + verifyUpdates( updates ); + } + + @Test + public void successfulCloseMustCloseGBPTree() throws Exception + { + // given + populator.create(); + Optional existingMapping = pageCache.getExistingMapping( indexFile ); + if ( existingMapping.isPresent() ) + { + existingMapping.get().close(); + } + else + { + fail( "Expected underlying GBPTree to have a mapping for this file" ); + } + + // when + populator.close( true ); + + // then + existingMapping = pageCache.getExistingMapping( indexFile ); + assertFalse( existingMapping.isPresent() ); + } + + @Test + public void successfulCloseMustMarkIndexAsOnline() throws Exception + { + // given + populator.create(); + + // when + populator.close( true ); + + // then + assertHeader( true, null, false ); + } + + @Test + public void unsuccessfulCloseMustSucceedWithoutMarkAsFailed() throws Exception + { + // given + populator.create(); + + // then + populator.close( false ); + } + + @Test + public void unsuccessfulCloseMustCloseGBPTree() throws Exception + { + // given + populator.create(); + Optional existingMapping = pageCache.getExistingMapping( indexFile ); + if ( existingMapping.isPresent() ) + { + existingMapping.get().close(); + } + else + { + fail( "Expected underlying GBPTree to have a mapping for this file" ); + } + + // when + populator.close( false ); + + // then + existingMapping = pageCache.getExistingMapping( indexFile ); + assertFalse( existingMapping.isPresent() ); + } + + @Test + public void unsuccessfulCloseMustNotMarkIndexAsOnline() throws Exception + { + // given + populator.create(); + + // when + populator.close( false ); + + // then + assertHeader( false, "", false ); + } + + @Test + public void closeMustWriteFailureMessageAfterMarkedAsFailed() throws Exception + { + // given + populator.create(); + + // when + String failureMessage = "Fly, you fools!"; + populator.markAsFailed( failureMessage ); + populator.close( false ); + + // then + assertHeader( false, failureMessage, false ); + } + + @Test + public void closeMustWriteFailureMessageAfterMarkedAsFailedWithLongMessage() throws Exception + { + // given + populator.create(); + + // when + String failureMessage = longString( pageCache.pageSize() ); + populator.markAsFailed( failureMessage ); + populator.close( false ); + + // then + assertHeader( false, failureMessage, true ); + } + + @Test + public void successfulCloseMustThrowIfMarkedAsFailed() throws Exception + { + // given + populator.create(); + + // when + populator.markAsFailed( "" ); + + // then + try + { + populator.close( true ); + fail( "Expected successful close to fail after markedAsFailed" ); + } + catch ( IllegalStateException e ) + { + // good + } + populator.close( false ); + } + + @Test + public void shouldApplyLargeAmountOfInterleavedRandomUpdates() throws Exception + { + // given + populator.create(); + random.reset(); + Random updaterRandom = new Random( random.seed() ); + Iterator> updates = randomUniqueUpdateGenerator( random, 0 ); + int numberOfPopulatorUpdates = LARGE_AMOUNT_OF_UPDATES; + + // when + int count = 0; + for ( int i = 0; i < numberOfPopulatorUpdates; i++ ) + { + if ( updaterRandom.nextFloat() < 0.1 ) + { + try ( IndexUpdater indexUpdater = populator.newPopulatingUpdater( null_property_accessor ) ) + { + int numberOfUpdaterUpdates = updaterRandom.nextInt( 100 ); + for ( int j = 0; j < numberOfUpdaterUpdates; j++ ) + { + indexUpdater.process( updates.next() ); + count++; + } + } + } + populator.add( updates.next() ); + count++; + } + + // then + populator.close( true ); + random.reset(); + verifyUpdates( randomUniqueUpdateGenerator( random, 0 ), count ); + } + + protected Iterator> randomUniqueUpdateGenerator( RandomRule randomRule, + float fractionDuplicates ) + { + return new PrefetchingIterator>() + { + private final Set uniqueCompareValues = new HashSet<>(); + private final List uniqueValues = new ArrayList<>(); + private long currentEntityId; + + @Override + protected IndexEntryUpdate fetchNextOrNull() + { + Number value; + if ( fractionDuplicates > 0 && !uniqueValues.isEmpty() && + randomRule.nextFloat() < fractionDuplicates ) + { + value = existingNonUniqueValue( randomRule ); + } + else + { + value = newUniqueValue( randomRule ); + } + + return add( currentEntityId++, value ); + } + + private Number newUniqueValue( RandomRule randomRule ) + { + Number value; + Double compareValue; + do + { + value = randomRule.numberPropertyValue(); + compareValue = value.doubleValue(); + } + while ( !uniqueCompareValues.add( compareValue ) ); + uniqueValues.add( value ); + return value; + } + + private Number existingNonUniqueValue( RandomRule randomRule ) + { + return uniqueValues.get( randomRule.nextInt( uniqueValues.size() ) ); + } + }; + } + + @SuppressWarnings( "rawtypes" ) + // unique -> + // addShouldThrowOnDuplicateValues + // updaterShouldThrowOnDuplicateValues + // non-unique -> + // addShouldApplyDuplicateValues + // updaterShouldApplyDuplicateValues + + // successfulCloseMustCloseGBPTree + // successfulCloseMustCheckpointGBPTree (already verified by add / updater tests) + // successfulCloseMustMarkIndexAsOnline + + // unsuccessfulCloseMustCloseGBPTree + // unsuccessfulCloseMustNotMarkIndexAsOnline + // unsuccessfulCloseMustSucceedWithoutMarkAsFailed + + // closeMustWriteFailureMessageAfterMarkedAsFailed + // closeMustWriteFailureMessageAfterMarkedAsFailedWithLongMessage + // successfulCloseMustThrowIfMarkedAsFailed + + // shouldApplyLargeAmountOfInterleavedRandomUpdates + // unique -> + // shouldThrowOnLargeAmountOfInterleavedRandomUpdatesWithDuplicates + // non-unique -> + // shouldApplyLargeAmountOfInterleavedRandomUpdatesWithDuplicates + + // SAMPLING + // includeSample + // configureSampling + // sampleResult + + // ??? + // todo closeAfterDrop + // todo dropAfterClose + + // METHODS + // create() + // drop() + // add( Collection> updates ) + // add( IndexEntryUpdate update ) + // verifyDeferredConstraints( PropertyAccessor propertyAccessor ) + // newPopulatingUpdater( PropertyAccessor accessor ) + // close( boolean populationCompletedSuccessfully ) + // markAsFailed( String failure ) + + static IndexEntryUpdate[] someIndexEntryUpdates() + { + return new IndexEntryUpdate[]{ + add( 0, 0 ), + add( 1, 4 ), + add( 2, Double.MAX_VALUE ), + add( 3, -Double.MAX_VALUE ), + add( 4, Float.MAX_VALUE ), + add( 5, -Float.MAX_VALUE ), + add( 6, Long.MAX_VALUE ), + add( 7, Long.MIN_VALUE ), + add( 8, Integer.MAX_VALUE ), + add( 9, Integer.MIN_VALUE ), + add( 10, Short.MAX_VALUE ), + add( 11, Short.MIN_VALUE ), + add( 12, Byte.MAX_VALUE ), + add( 13, Byte.MIN_VALUE ) + }; + } + + @SuppressWarnings( "rawtypes" ) + static IndexEntryUpdate[] someDuplicateIndexEntryUpdates() + { + return new IndexEntryUpdate[]{ + add( 0, 0 ), + add( 1, 4 ), + add( 2, Double.MAX_VALUE ), + add( 3, -Double.MAX_VALUE ), + add( 4, Float.MAX_VALUE ), + add( 5, -Float.MAX_VALUE ), + add( 6, Long.MAX_VALUE ), + add( 7, Long.MIN_VALUE ), + add( 8, Integer.MAX_VALUE ), + add( 9, Integer.MIN_VALUE ), + add( 10, Short.MAX_VALUE ), + add( 11, Short.MIN_VALUE ), + add( 12, Byte.MAX_VALUE ), + add( 13, Byte.MIN_VALUE ), + add( 14, 0 ), + add( 15, 4 ), + add( 16, Double.MAX_VALUE ), + add( 17, -Double.MAX_VALUE ), + add( 18, Float.MAX_VALUE ), + add( 19, -Float.MAX_VALUE ), + add( 20, Long.MAX_VALUE ), + add( 21, Long.MIN_VALUE ), + add( 22, Integer.MAX_VALUE ), + add( 23, Integer.MIN_VALUE ), + add( 24, Short.MAX_VALUE ), + add( 25, Short.MIN_VALUE ), + add( 26, Byte.MAX_VALUE ), + add( 27, Byte.MIN_VALUE ) + }; + } + + private void assertHeader( boolean online, String failureMessage, boolean messageTruncated ) throws IOException + { + NativeSchemaIndexHeaderReader headerReader = new NativeSchemaIndexHeaderReader(); + try ( GBPTree tree = new GBPTree<>( pageCache, indexFile, layout, 0, GBPTree.NO_MONITOR, + headerReader, RecoveryCleanupWorkCollector.IMMEDIATE ) ) + { + if ( online ) + { + assertEquals( "Index was not marked as online when expected not to be.", BYTE_ONLINE, headerReader.state ); + assertNull( "Expected failure message to be null when marked as online.", headerReader.failureMessage ); + } + else + { + assertEquals( "Index was marked as online when expected not to be.", BYTE_FAILED, headerReader.state ); + if ( messageTruncated ) + { + assertTrue( headerReader.failureMessage.length() < failureMessage.length() ); + assertTrue( failureMessage.startsWith( headerReader.failureMessage ) ); + } + else + { + assertEquals( failureMessage, headerReader.failureMessage ); + } + } + } + } + + private String longString( int length ) + { + String alphabet = "123xyz"; + StringBuffer outputBuffer = new StringBuffer( length ); + for ( int i = 0; i < length; i++ ) + { + outputBuffer.append( alphabet.charAt( random.nextInt( alphabet.length() ) ) ); + } + return outputBuffer.toString(); + } + + void applyInterleaved( IndexEntryUpdate[] updates, IndexUpdater updater, + NativeSchemaIndexPopulator populator ) throws IOException, IndexEntryConflictException + { + for ( IndexEntryUpdate update : updates ) + { + if ( random.nextBoolean() ) + { + populator.add( update ); + } + else + { + updater.process( update ); + } + } + } + + protected void verifyUpdates( Iterator> indexEntryUpdateIterator, int count ) + throws IOException + { + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] updates = new IndexEntryUpdate[count]; + for ( int i = 0; i < count; i++ ) + { + updates[i] = indexEntryUpdateIterator.next(); + } + verifyUpdates( updates ); + } + + @SuppressWarnings( "unchecked" ) + void verifyUpdates( IndexEntryUpdate[] updates ) + throws IOException + { + Hit[] expectedHits = convertToHits( updates, layout ); + List> actualHits = new ArrayList<>(); + try ( GBPTree tree = new GBPTree<>( pageCache, indexFile, layout, 0, GBPTree.NO_MONITOR, + GBPTree.NO_HEADER, RecoveryCleanupWorkCollector.IMMEDIATE ); + RawCursor,IOException> scan = scan( tree ) ) + { + while ( scan.next() ) + { + actualHits.add( deepCopy( scan.get() ) ); + } + } + + Comparator> hitComparator = ( h1, h2 ) -> + { + int keyCompare = layout.compare( h1.key(), h2.key() ); + if ( keyCompare == 0 ) + { + return compareValue( h1.value(), h2.value() ); + } + else + { + return keyCompare; + } + }; + assertSameHits( expectedHits, actualHits.toArray( new Hit[0] ), hitComparator ); + } + + protected abstract int compareValue( VALUE value1, VALUE value2 ); + + int compareIndexedPropertyValue( SchemaNumberValue value1, SchemaNumberValue value2 ) + { + int typeCompare = Byte.compare( value1.type(), value2.type() ); + if ( typeCompare == 0 ) + { + switch ( value1.type() ) + { + case LONG: + return Long.compare( value1.rawValueBits(), value2.rawValueBits() ); + case FLOAT: + return Float.compare( + Float.intBitsToFloat( (int) value1.rawValueBits() ), + Float.intBitsToFloat( (int) value2.rawValueBits() ) ); + case DOUBLE: + return Double.compare( + Double.longBitsToDouble( value1.rawValueBits() ), + Double.longBitsToDouble( value2.rawValueBits() ) ); + default: + throw new IllegalArgumentException( + "Expected type to be LONG, FLOAT or DOUBLE (" + LONG + "," + FLOAT + "," + DOUBLE + + "). But was " + value1.type() ); + } + } + return typeCompare; + } + + private void assertSameHits( Hit[] expectedHits, Hit[] actualHits, + Comparator> comparator ) + { + Arrays.sort( expectedHits, comparator ); + Arrays.sort( actualHits, comparator ); + assertEquals( "Array length differ", expectedHits.length, actualHits.length ); + + for ( int i = 0; i < expectedHits.length; i++ ) + { + Hit expected = expectedHits[i]; + Hit actual = actualHits[i]; + assertTrue( "Hits differ on item number " + i + ". Expected " + expected + " but was " + actual, + comparator.compare( expected, actual ) == 0 ); + } + } + + private Hit deepCopy( Hit from ) + { + KEY intoKey = layout.newKey(); + VALUE intoValue = layout.newValue(); + layout.copyKey( from.key(), intoKey ); + copyValue( from.value(), intoValue ); + return new SimpleHit( intoKey, intoValue ); + } + + protected abstract void copyValue( VALUE value, VALUE intoValue ); + + @SuppressWarnings( "unchecked" ) + private Hit[] convertToHits( IndexEntryUpdate[] updates, + Layout layout ) + { + List> hits = new ArrayList<>( updates.length ); + for ( IndexEntryUpdate u : updates ) + { + KEY key = layout.newKey(); + key.from( u.getEntityId(), u.values() ); + VALUE value = layout.newValue(); + value.from( u.getEntityId(), u.values() ); + hits.add( hit( key, value ) ); + } + return hits.toArray( new Hit[0] ); + } + + private Hit hit( final KEY key, final VALUE value ) + { + return new SimpleHit( key, value ); + } + + private static class NativeSchemaIndexHeaderReader implements Header.Reader + { + byte state; + String failureMessage; + + @Override + public void read( PageCursor from, int length ) + { + state = from.getByte(); + if ( state == NativeSchemaIndexPopulator.BYTE_FAILED ) + { + short messageLength = from.getShort(); + byte[] failureMessageBytes = new byte[messageLength]; + from.getBytes( failureMessageBytes ); + failureMessage = new String( failureMessageBytes, Charsets.UTF_8 ); + } + } + } + + private class SimpleHit implements Hit + { + private final KEY key; + private final VALUE value; + + SimpleHit( KEY key, VALUE value ) + { + this.key = key; + this.value = value; + } + + @Override + public KEY key() + { + return key; + } + + @Override + public VALUE value() + { + return value; + } + + @Override + public boolean equals( Object o ) + { + if ( this == o ) + { + return true; + } + if ( o == null || getClass() != o.getClass() ) + { + return false; + } + @SuppressWarnings( "unchecked" ) + Hit simpleHit = (Hit) o; + return Objects.equals( key(), simpleHit.key() ) && + Objects.equals( value, simpleHit.value() ); + } + + @Override + public int hashCode() + { + return Objects.hash( key, value ); + } + + @Override + public String toString() + { + return "[" + key + "," + value + "]"; + } + } + + private RawCursor, IOException> scan( GBPTree tree ) throws IOException + { +// tree.printTree( false, false, false ); + KEY lowest = layout.newKey(); + lowest.initAsLowest(); + KEY highest = layout.newKey(); + highest.initAsHighest(); + return tree.seek( lowest, highest ); + } + + protected static IndexEntryUpdate add( long nodeId, Object value ) + { + return IndexEntryUpdate.add( nodeId, indexDescriptor, value ); + } + + private byte[] fileWithContent() throws IOException + { + int size = 1000; + try ( StoreChannel storeChannel = fs.create( indexFile ) ) + { + byte[] someBytes = new byte[size]; + random.nextBytes( someBytes ); + storeChannel.writeAll( ByteBuffer.wrap( someBytes ) ); + return someBytes; + } + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NonUniqueNativeSchemaIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NonUniqueNativeSchemaIndexPopulatorTest.java new file mode 100644 index 0000000000000..aacb1ca6fb02c --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/NonUniqueNativeSchemaIndexPopulatorTest.java @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Random; + +import org.neo4j.index.internal.gbptree.Layout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.api.index.IndexEntryUpdate; +import org.neo4j.kernel.api.index.IndexUpdater; +import org.neo4j.kernel.api.schema.index.IndexDescriptor; +import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; +import org.neo4j.storageengine.api.schema.IndexSample; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import static java.util.Arrays.asList; + +import static org.neo4j.helpers.ArrayUtil.array; +import static org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector.IMMEDIATE; +import static org.neo4j.kernel.impl.index.schema.FullScanNonUniqueIndexSamplerTest.countUniqueValues; + +public class NonUniqueNativeSchemaIndexPopulatorTest + extends NativeSchemaIndexPopulatorTest +{ + @Override + Layout createLayout() + { + return new NonUniqueSchemaNumberIndexLayout(); + } + + @Override + NativeSchemaIndexPopulator createPopulator( PageCache pageCache, File indexFile, + Layout layout, IndexSamplingConfig samplingConfig ) + { + return new NonUniqueNativeSchemaIndexPopulator<>( pageCache, indexFile, layout, IMMEDIATE, samplingConfig ); + } + + @Override + protected int compareValue( NonUniqueSchemaNumberValue value1, NonUniqueSchemaNumberValue value2 ) + { + return compareIndexedPropertyValue( value1, value2 ); + } + + @Test + public void addShouldApplyDuplicateValues() throws Exception + { + // given + populator.create(); + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] updates = someDuplicateIndexEntryUpdates(); + + // when + populator.add( Arrays.asList( updates ) ); + + // then + populator.close( true ); + verifyUpdates( updates ); + } + + @Test + public void updaterShouldApplyDuplicateValues() throws Exception + { + // given + populator.create(); + IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ); + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] updates = someDuplicateIndexEntryUpdates(); + + // when + for ( IndexEntryUpdate update : updates ) + { + updater.process( update ); + } + + // then + populator.close( true ); + verifyUpdates( updates ); + } + + @Test + public void shouldApplyLargeAmountOfInterleavedRandomUpdatesWithDuplicates() throws Exception + { + // given + populator.create(); + random.reset(); + Random updaterRandom = new Random( random.seed() ); + Iterator> updates = randomUniqueUpdateGenerator( random, 0.1f ); + int numberOfPopulatorUpdates = LARGE_AMOUNT_OF_UPDATES; + + // when + int count = 0; + for ( int i = 0; i < numberOfPopulatorUpdates; i++ ) + { + if ( updaterRandom.nextFloat() < 0.1 ) + { + try ( IndexUpdater indexUpdater = populator.newPopulatingUpdater( null_property_accessor ) ) + { + int numberOfUpdaterUpdates = updaterRandom.nextInt( 100 ); + for ( int j = 0; j < numberOfUpdaterUpdates; j++ ) + { + indexUpdater.process( updates.next() ); + count++; + } + } + } + populator.add( updates.next() ); + count++; + } + + // then + populator.close( true ); + random.reset(); + verifyUpdates( randomUniqueUpdateGenerator( random, 0.1f ), count ); + } + + @Test + public void shouldFailOnSampleBeforeConfiguredSampling() throws Exception + { + // GIVEN + populator.create(); + + // WHEN + try + { + populator.sampleResult(); + fail(); + } + catch ( IllegalStateException e ) + { + // THEN good + } + populator.close( true ); + } + + @Test + public void shouldSampleWholeIndexIfConfiguredForPopulatingSampling() throws Exception + { + // GIVEN + populator.create(); + populator.configureSampling( false ); + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] updates = someIndexEntryUpdates(); + populator.add( Arrays.asList( updates ) ); + + // WHEN + IndexSample sample = populator.sampleResult(); + + // THEN + assertEquals( updates.length, sample.sampleSize() ); + assertEquals( updates.length, sample.uniqueValues() ); + assertEquals( updates.length, sample.indexSize() ); + populator.close( true ); + } + + @Test + public void shouldSampleUpdatesIfConfiguredForOnlineSampling() throws Exception + { + // GIVEN + populator.create(); + populator.configureSampling( true ); + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] scanUpdates = someIndexEntryUpdates(); + populator.add( Arrays.asList( scanUpdates ) ); + Number[] updates = array( 101, 102, 102, 103, 103 ); + try ( IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ) ) + { + long nodeId = 1000; + for ( Number number : updates ) + { + IndexEntryUpdate update = add( nodeId++, number ); + updater.process( update ); + populator.includeSample( update ); + } + } + + // WHEN + IndexSample sample = populator.sampleResult(); + + // THEN + assertEquals( updates.length, sample.sampleSize() ); + assertEquals( countUniqueValues( asList( updates ) ), sample.uniqueValues() ); + assertEquals( updates.length, sample.indexSize() ); + populator.close( true ); + } + + @Override + protected void copyValue( NonUniqueSchemaNumberValue value, NonUniqueSchemaNumberValue intoValue ) + { + intoValue.type = value.type; + intoValue.rawValueBits = value.rawValueBits; + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/UniqueNativeSchemaIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/UniqueNativeSchemaIndexPopulatorTest.java new file mode 100644 index 0000000000000..0d8540e35fe8f --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/UniqueNativeSchemaIndexPopulatorTest.java @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Random; + +import org.neo4j.index.internal.gbptree.Layout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; +import org.neo4j.kernel.api.index.IndexEntryUpdate; +import org.neo4j.kernel.api.index.IndexUpdater; +import org.neo4j.kernel.api.schema.index.IndexDescriptor; +import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; +import org.neo4j.storageengine.api.schema.IndexSample; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import static org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector.IMMEDIATE; + +public class UniqueNativeSchemaIndexPopulatorTest + extends NativeSchemaIndexPopulatorTest +{ + @Override + NativeSchemaIndexPopulator createPopulator( + PageCache pageCache, File indexFile, + Layout layout, IndexSamplingConfig samplingConfig ) + { + return new UniqueNativeSchemaIndexPopulator<>( pageCache, indexFile, layout, IMMEDIATE ); + } + + @Override + protected int compareValue( UniqueSchemaNumberValue value1, UniqueSchemaNumberValue value2 ) + { + int valueCompare = compareIndexedPropertyValue( value1, value2 ); + if ( valueCompare == 0 ) + { + return Long.compare( value1.getEntityId(), value2.getEntityId() ); + } + return valueCompare; + } + + @Override + Layout createLayout() + { + return new UniqueSchemaNumberIndexLayout(); + } + + @Test + public void addShouldThrowOnDuplicateValues() throws Exception + { + // given + populator.create(); + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] updates = someDuplicateIndexEntryUpdates(); + + // when + try + { + populator.add( Arrays.asList( updates ) ); + fail( "Updates should have conflicted" ); + } + catch ( IndexEntryConflictException e ) + { + // then good + } + + populator.close( true ); + } + + @Test + public void updaterShouldThrowOnDuplicateValues() throws Exception + { + // given + populator.create(); + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] updates = someDuplicateIndexEntryUpdates(); + try ( IndexUpdater updater = populator.newPopulatingUpdater( null_property_accessor ) ) + { + // when + for ( IndexEntryUpdate update : updates ) + { + updater.process( update ); + } + fail( "Updates should have conflicted" ); + } + catch ( IndexEntryConflictException e ) + { + // then good + } + + populator.close( true ); + } + + @Test + public void shouldThrowOnLargeAmountOfInterleavedRandomUpdatesWithDuplicates() throws Exception + { + // given + populator.create(); + random.reset(); + Random updaterRandom = new Random( random.seed() ); + Iterator> updates = randomUniqueUpdateGenerator( random, 0.01f ); + int numberOfPopulatorUpdates = LARGE_AMOUNT_OF_UPDATES; + Number failSafeDuplicateValue = 12345.6789D; + + // when + try + { + populator.add( add( 1_000_000_000, failSafeDuplicateValue ) ); + for ( int i = 0; i < numberOfPopulatorUpdates; i++ ) + { + if ( updaterRandom.nextFloat() < 0.1 ) + { + try ( IndexUpdater indexUpdater = populator.newPopulatingUpdater( null_property_accessor ) ) + { + int numberOfUpdaterUpdates = updaterRandom.nextInt( 100 ); + for ( int j = 0; j < numberOfUpdaterUpdates; j++ ) + { + indexUpdater.process( updates.next() ); + } + } + } + populator.add( updates.next() ); + } + populator.add( add( 1_000_000_001, failSafeDuplicateValue ) ); + fail( "Should have bumped into and detected duplicate" ); + } + catch ( IndexEntryConflictException e ) + { + // then good + } + + populator.close( true ); + } + + @Test + public void shouldSampleUpdates() throws Exception + { + // GIVEN + populator.create(); + populator.configureSampling( true ); // has no effect, really + @SuppressWarnings( "unchecked" ) + IndexEntryUpdate[] updates = someIndexEntryUpdates(); + + // WHEN + for ( IndexEntryUpdate update : updates ) + { + populator.add( update ); + populator.includeSample( update ); + } + IndexSample sample = populator.sampleResult(); + + // THEN + assertEquals( updates.length, sample.sampleSize() ); + assertEquals( updates.length, sample.uniqueValues() ); + assertEquals( updates.length, sample.indexSize() ); + populator.close( true ); + } + + + @Override + protected void copyValue( UniqueSchemaNumberValue value, UniqueSchemaNumberValue into ) + { + into.type = value.type; + into.rawValueBits = value.rawValueBits; + into.entityId = value.entityId; + } +} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/DirectNonUniqueIndexSampler.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/DirectNonUniqueIndexSampler.java index 5736b0525cdc0..ce2787a47e03c 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/DirectNonUniqueIndexSampler.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/DirectNonUniqueIndexSampler.java @@ -32,9 +32,8 @@ * Non unique index sampler that ignores all include/exclude calls and builds * sample based on values obtained directly from targeted index. */ -public class DirectNonUniqueIndexSampler implements NonUniqueIndexSampler +public class DirectNonUniqueIndexSampler extends NonUniqueIndexSampler.Adapter { - private SchemaIndex luceneIndex; public DirectNonUniqueIndexSampler( SchemaIndex luceneIndex ) @@ -42,30 +41,6 @@ public DirectNonUniqueIndexSampler( SchemaIndex luceneIndex ) this.luceneIndex = luceneIndex; } - @Override - public void include( String value ) - { - // no-op - } - - @Override - public void include( String value, long increment ) - { - // no-op - } - - @Override - public void exclude( String value ) - { - // no-op - } - - @Override - public void exclude( String value, long decrement ) - { - // no-op - } - @Override public IndexSample result() {