From cb9c289dfa6670c82d8d6abddbdebef392474f6f Mon Sep 17 00:00:00 2001 From: Mattias Persson Date: Tue, 27 Jun 2017 12:41:26 +0200 Subject: [PATCH] Simplified and unified NodeLabelRange implementation Previously each label scan store had its own implementation of NodeLabelRange as well as parsing and composition of them. The Lucene implementation was garbage-heavy and didn't have sorted node ids within each range, which made implementing the gap-free label range iterator difficult. There have been clarifications and javadocs added around what the NodeLabelRange methods mean. Also the return values better fit its use case, which is for the consistency checker. --- .../GapFreeAllEntriesLabelScanReader.java | 118 ++++------- .../GapFreeAllEntriesLabelScanReaderTest.java | 188 ++++++++++++++++++ .../report/ConsistencyReporterTest.java | 4 +- .../labelscan/AllEntriesLabelScanReader.java | 8 + .../kernel/api/labelscan/NodeLabelRange.java | 112 ++++++++++- .../NativeAllEntriesLabelScanReader.java | 103 ++-------- .../impl/labelscan/LabelScanStoreTest.java | 29 ++- .../api/labelscan/NodeLabelRangeTest.java | 81 ++++++++ .../NativeAllEntriesLabelScanReaderTest.java | 12 -- .../impl/labelscan/LuceneNodeLabelRange.java | 114 ----------- .../LuceneAllEntriesLabelScanReader.java | 46 ++--- .../labelscan/LuceneNodeLabelRangeTest.java | 47 ----- 12 files changed, 483 insertions(+), 379 deletions(-) create mode 100644 community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/GapFreeAllEntriesLabelScanReaderTest.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/api/labelscan/NodeLabelRangeTest.java delete mode 100644 community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/labelscan/LuceneNodeLabelRange.java delete mode 100644 community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/labelscan/LuceneNodeLabelRangeTest.java diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/GapFreeAllEntriesLabelScanReader.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/GapFreeAllEntriesLabelScanReader.java index 669b3509a3beb..bbd9102bf217e 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/GapFreeAllEntriesLabelScanReader.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/GapFreeAllEntriesLabelScanReader.java @@ -25,6 +25,10 @@ import org.neo4j.kernel.api.labelscan.AllEntriesLabelScanReader; import org.neo4j.kernel.api.labelscan.NodeLabelRange; +/** + * Inserts empty {@link NodeLabelRange} for those ranges missing from the source iterator. + * High node id is known up front such that ranges are returned up to that point. + */ class GapFreeAllEntriesLabelScanReader implements AllEntriesLabelScanReader { private final AllEntriesLabelScanReader nodeLabelRanges; @@ -48,116 +52,74 @@ public void close() throws Exception nodeLabelRanges.close(); } + @Override + public int rangeSize() + { + return nodeLabelRanges.rangeSize(); + } + @Override public Iterator iterator() { - return new GapFillingIterator( nodeLabelRanges.iterator(), highId ); + return new GapFillingIterator( nodeLabelRanges.iterator(), (highId - 1) / nodeLabelRanges.rangeSize(), + nodeLabelRanges.rangeSize() ); } private static class GapFillingIterator extends PrefetchingIterator { - private static final int BATCH_SIZE = 1_000; - private static final long[] EMPTY_LONG_ARRAY = new long[0]; - private final long highId; - private Iterator source; + private final long highestRangeId; + private final Iterator source; + private final long[][] emptyRangeData; + private NodeLabelRange nextFromSource; - private long[] sourceNodeIds; - private int sourceIndex; - private int currentId; - private boolean first; + private long currentRangeId = -1; - GapFillingIterator( Iterator nodeLableRangeIterator, long highId ) + GapFillingIterator( Iterator nodeLableRangeIterator, long highestRangeId, int rangeSize ) { - this.highId = highId; + this.highestRangeId = highestRangeId; this.source = nodeLableRangeIterator; - this.first = true; + this.emptyRangeData = new long[rangeSize][]; } @Override protected NodeLabelRange fetchNextOrNull() { - long baseId = currentId; - int batchSize = BATCH_SIZE; - long[] nodes = new long[batchSize]; - long[][] labels = new long[batchSize][]; - - int cursor = 0; - for ( ; cursor < batchSize; cursor++, currentId++ ) + while ( true ) { - // First or empty source - if ( first || (sourceNodeIds != null && sourceIndex >= sourceNodeIds.length) ) + // These conditions only come into play after we've gotten the first range from the source + if ( nextFromSource != null ) { - first = false; - if ( source.hasNext() ) + if ( currentRangeId + 1 == nextFromSource.id() ) { - nextFromSource = source.next(); - sourceNodeIds = nextFromSource.nodes(); - sourceIndex = 0; + // Next to return is the one from source + currentRangeId++; + return nextFromSource; } - else + + if ( currentRangeId < nextFromSource.id() ) { - nextFromSource = null; - sourceNodeIds = null; + // Source range iterator has a gap we need to fill + return new NodeLabelRange( ++currentRangeId, emptyRangeData ); } } - if ( currentId >= highId && sourceNodeIds == null ) + if ( source.hasNext() ) { - break; + // The source iterator has more ranges, grab the next one + nextFromSource = source.next(); + // continue in the outer loop } - - nodes[cursor] = currentId; - if ( sourceNodeIds != null && sourceNodeIds[sourceIndex] == currentId ) + else if ( currentRangeId < highestRangeId ) { - labels[cursor] = nextFromSource.labels( currentId ); - sourceIndex++; + nextFromSource = new NodeLabelRange( highestRangeId, emptyRangeData ); + // continue in the outer loop } else { - labels[cursor] = EMPTY_LONG_ARRAY; + // End has been reached + return null; } } - return cursor > 0 ? new SimpleNodeLabelRange( baseId, nodes, labels ) : null; - } - - private static class SimpleNodeLabelRange extends NodeLabelRange - { - private final long baseId; - private final long[] nodes; - private final long[][] labels; - - SimpleNodeLabelRange( long baseId, long[] nodes, long[][] labels ) - { - this.baseId = baseId; - this.nodes = nodes; - this.labels = labels; - } - - @Override - public int id() - { - return (int) baseId; - } - - @Override - public long[] nodes() - { - return nodes; - } - - @Override - public long[] labels( long nodeId ) - { - return labels[(int) (nodeId - baseId)]; - } - - @Override - public String toString() - { - String rangeString = baseId + "-" + (baseId + nodes.length); - String prefix = "NodeLabelRange[idRange=" + rangeString; - return toString( prefix, nodes, labels ); - } } } } diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/GapFreeAllEntriesLabelScanReaderTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/GapFreeAllEntriesLabelScanReaderTest.java new file mode 100644 index 0000000000000..473d9389ef6f3 --- /dev/null +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/GapFreeAllEntriesLabelScanReaderTest.java @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.consistency.checking.full; + +import org.junit.Rule; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.neo4j.kernel.api.labelscan.AllEntriesLabelScanReader; +import org.neo4j.kernel.api.labelscan.NodeLabelRange; +import org.neo4j.test.rule.RandomRule; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import static org.neo4j.collection.primitive.PrimitiveLongCollections.EMPTY_LONG_ARRAY; + +public class GapFreeAllEntriesLabelScanReaderTest +{ + private static final int EMPTY_RANGE = 0; + private static final int NON_EMPTY_RANGE = 0b10101; // 0, 2, 4 + private static final int RANGE_SIZE = 10; + private static final long[] LABEL_IDS = new long[] {1}; + + @Rule + public final RandomRule random = new RandomRule(); + + @Test + public void shouldFillGapInBeginning() throws Exception + { + // given + int[] ranges = array( EMPTY_RANGE, EMPTY_RANGE, NON_EMPTY_RANGE ); + GapFreeAllEntriesLabelScanReader reader = newGapFreeAllEntriesLabelScanReader( ranges ); + + // when + Iterator iterator = reader.iterator(); + + // then + assertRanges( iterator, ranges ); + } + + @Test + public void shouldFillGapInEnd() throws Exception + { + // given + int[] ranges = array( NON_EMPTY_RANGE, EMPTY_RANGE, EMPTY_RANGE ); + GapFreeAllEntriesLabelScanReader reader = newGapFreeAllEntriesLabelScanReader( ranges ); + + // when + Iterator iterator = reader.iterator(); + + // then + assertRanges( iterator, ranges ); + } + + @Test + public void shouldFillGapInMiddle() throws Exception + { + // given + int[] ranges = array( EMPTY_RANGE, NON_EMPTY_RANGE, EMPTY_RANGE ); + GapFreeAllEntriesLabelScanReader reader = newGapFreeAllEntriesLabelScanReader( ranges ); + + // when + Iterator iterator = reader.iterator(); + + // then + assertRanges( iterator, ranges ); + } + + @Test + public void shouldFillRandomGaps() throws Exception + { + // given + int numberOfRanges = random.intBetween( 50, 100 ); + int[] ranges = new int[numberOfRanges]; + for ( int rangeId = 0; rangeId < numberOfRanges; rangeId++ ) + { + ranges[rangeId] = random.nextInt( 1 << RANGE_SIZE ); + } + GapFreeAllEntriesLabelScanReader reader = newGapFreeAllEntriesLabelScanReader( ranges ); + + // when + Iterator iterator = reader.iterator(); + + // then + assertRanges( iterator, ranges ); + } + + private void assertRanges( Iterator iterator, int[] expectedRanges ) + { + for ( int expectedRangeId = 0; expectedRangeId < expectedRanges.length; expectedRangeId++ ) + { + assertTrue( iterator.hasNext() ); + NodeLabelRange actualRange = iterator.next(); + assertEquals( expectedRangeId, actualRange.id() ); + int expectedRange = expectedRanges[expectedRangeId]; + long baseNodeId = expectedRangeId * RANGE_SIZE; + for ( int i = 0; i < RANGE_SIZE; i++ ) + { + long nodeId = baseNodeId + i; + long[] expectedLabelIds = (expectedRange & (1 << i)) == 0 ? EMPTY_LONG_ARRAY : LABEL_IDS; + assertArrayEquals( expectedLabelIds, actualRange.labels( nodeId ) ); + assertEquals( nodeId, actualRange.nodes()[i] ); + } + } + assertFalse( iterator.hasNext() ); + } + + private GapFreeAllEntriesLabelScanReader newGapFreeAllEntriesLabelScanReader( int... ranges ) + { + return new GapFreeAllEntriesLabelScanReader( ranges( RANGE_SIZE, ranges ), RANGE_SIZE * ranges.length ); + } + + private static AllEntriesLabelScanReader ranges( int rangeSize, int... ranges ) + { + List rangeList = new ArrayList<>(); + for ( int rangeId = 0; rangeId < ranges.length; rangeId++ ) + { + rangeList.add( new NodeLabelRange( rangeId, labelsPerNode( ranges[rangeId] ) ) ); + } + + return new AllEntriesLabelScanReader() + { + @Override + public void close() throws Exception + { // Nothing to close + } + + @Override + public Iterator iterator() + { + return rangeList.iterator(); + } + + @Override + public long maxCount() + { + return ranges.length * rangeSize; + } + + @Override + public int rangeSize() + { + return RANGE_SIZE; + } + }; + } + + private static long[][] labelsPerNode( int relativeNodeIds ) + { + long[][] result = new long[RANGE_SIZE][]; + for ( int i = 0; i < RANGE_SIZE; i++ ) + { + if ( (relativeNodeIds & (1 << i)) != 0 ) + { + result[i] = LABEL_IDS; + } + } + return result; + } + + private static int[] array( int... relativeNodeIds ) + { + return relativeNodeIds; + } +} diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/report/ConsistencyReporterTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/report/ConsistencyReporterTest.java index 29e1e1a24849a..f5e48e809cd64 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/report/ConsistencyReporterTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/report/ConsistencyReporterTest.java @@ -51,8 +51,8 @@ import org.neo4j.consistency.store.synthetic.CountsEntry; import org.neo4j.consistency.store.synthetic.IndexEntry; import org.neo4j.consistency.store.synthetic.LabelScanDocument; -import org.neo4j.kernel.api.impl.labelscan.LuceneNodeLabelRange; import org.neo4j.kernel.api.index.SchemaIndexProvider; +import org.neo4j.kernel.api.labelscan.NodeLabelRange; import org.neo4j.kernel.api.schema.index.IndexDescriptorFactory; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; import org.neo4j.kernel.impl.store.record.DynamicRecord; @@ -379,7 +379,7 @@ private Object parameter( Class type ) } if ( type == LabelScanDocument.class ) { - return new LabelScanDocument( new LuceneNodeLabelRange( 0, new long[] {}, new long[][] {} ) ); + return new LabelScanDocument( new NodeLabelRange( 0, new long[][] {} ) ); } if ( type == IndexEntry.class ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/labelscan/AllEntriesLabelScanReader.java b/community/kernel/src/main/java/org/neo4j/kernel/api/labelscan/AllEntriesLabelScanReader.java index 3740adf7d766c..86b2b291ba29c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/api/labelscan/AllEntriesLabelScanReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/labelscan/AllEntriesLabelScanReader.java @@ -21,6 +21,14 @@ import org.neo4j.helpers.collection.BoundedIterable; +/** + * Iterates over all label data in a label index. + */ public interface AllEntriesLabelScanReader extends BoundedIterable { + /** + * @return size of a range. All {@link NodeLabelRange} instances handed out by this iterator + * has the same range size. + */ + int rangeSize(); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/labelscan/NodeLabelRange.java b/community/kernel/src/main/java/org/neo4j/kernel/api/labelscan/NodeLabelRange.java index 6460911120d50..b0883812c0db3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/api/labelscan/NodeLabelRange.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/api/labelscan/NodeLabelRange.java @@ -19,15 +19,81 @@ */ package org.neo4j.kernel.api.labelscan; -public abstract class NodeLabelRange +import java.util.ArrayList; +import java.util.List; + +import static java.lang.Math.toIntExact; + +import static org.neo4j.collection.primitive.PrimitiveLongCollections.EMPTY_LONG_ARRAY; +import static org.neo4j.collection.primitive.PrimitiveLongCollections.asArray; + +/** + * Represents a range of nodes and label ids attached to those nodes. All nodes in the range are present in + * {@link #nodes() nodes array}, but not all node ids will have corresponding {@link #labels(long) labels}, + * where an empty long[] will be returned instead. + */ +public class NodeLabelRange { - public abstract int id(); + private final long idRange; + private final long[] nodes; + private final long[][] labels; - public abstract long[] nodes(); + /** + * @param idRange node id range, e.g. in which id span the nodes are. + * @param labels long[][] where first dimension is relative node id in this range, i.e. 0-rangeSize + * and second the label ids for that node, potentially empty if there are none for that node. + * The first dimension must be the size of the range. + */ + public NodeLabelRange( long idRange, long[][] labels ) + { + this.idRange = idRange; + this.labels = labels; + int rangeSize = labels.length; + long baseNodeId = idRange * rangeSize; + + this.nodes = new long[rangeSize]; + for ( int i = 0; i < rangeSize; i++ ) + { + nodes[i] = baseNodeId + i; + } + } - public abstract long[] labels( long nodeId ); + /** + * @return the range id of this range. This is the base node id divided by range size. + * Example: A store with nodes 1,3,20,22 and a range size of 16 would return ranges: + * - rangeId=0, nodes=1,3 + * - rangeId=1, nodes=20,22 + */ + public long id() + { + return idRange; + } + + /** + * @return node ids in this range, the nodes in this array may or may not have {@link #labels(long) labels} + * attached to it. + */ + public long[] nodes() + { + return nodes; + } + + /** + * Returns the label ids (as longs) for the given node id. The {@code nodeId} must be one of the ids + * from {@link #nodes()}. + * + * @param nodeId the node id to return labels for. + * @return label ids for the given {@code nodeId}. + */ + public long[] labels( long nodeId ) + { + long firstNodeId = idRange * labels.length; + int index = toIntExact( nodeId - firstNodeId ); + assert index >= 0 && index < labels.length : "nodeId:" + nodeId + ", idRange:" + idRange; + return labels[index] != null ? labels[index] : EMPTY_LONG_ARRAY; + } - public String toString( String prefix, long[] nodes, long[][] labels ) + private static String toString( String prefix, long[] nodes, long[][] labels ) { StringBuilder result = new StringBuilder( prefix ); result.append( "; {" ); @@ -55,4 +121,40 @@ public String toString( String prefix, long[] nodes, long[][] labels ) } return result.append( "}]" ).toString(); } + + @Override + public String toString() + { + String rangeString = idRange * labels.length + "-" + (idRange + 1) * labels.length; + String prefix = "NodeLabelRange[idRange=" + rangeString; + return toString( prefix, nodes, labels ); + } + + public static void readBitmap( long bitmap, long labelId, List[] labelsPerNode ) + { + while ( bitmap != 0 ) + { + int relativeNodeId = Long.numberOfTrailingZeros( bitmap ); + if ( labelsPerNode[relativeNodeId] == null ) + { + labelsPerNode[relativeNodeId] = new ArrayList<>(); + } + labelsPerNode[relativeNodeId].add( labelId ); + bitmap &= bitmap - 1; + } + } + + public static long[][] convertState( List[] state ) + { + long[][] labelIdsByNodeIndex = new long[state.length][]; + for ( int i = 0; i < state.length; i++ ) + { + List labelIdList = state[i]; + if ( labelIdList != null ) + { + labelIdsByNodeIndex[i] = asArray( labelIdList.iterator() ); + } + } + return labelIdsByNodeIndex; + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeAllEntriesLabelScanReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeAllEntriesLabelScanReader.java index 5462465fc8e43..b42b138784685 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeAllEntriesLabelScanReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/labelscan/NativeAllEntriesLabelScanReader.java @@ -33,9 +33,10 @@ import org.neo4j.kernel.api.labelscan.NodeLabelRange; import static java.lang.Long.min; -import static java.lang.Math.toIntExact; import static java.util.Arrays.fill; -import static org.neo4j.collection.primitive.PrimitiveLongCollections.asArray; + +import static org.neo4j.kernel.api.labelscan.NodeLabelRange.convertState; +import static org.neo4j.kernel.api.labelscan.NodeLabelRange.readBitmap; import static org.neo4j.kernel.impl.index.labelscan.LabelScanValue.RANGE_SIZE; /** @@ -65,6 +66,12 @@ public long maxCount() return -1; } + @Override + public int rangeSize() + { + return RANGE_SIZE; + } + @Override public Iterator iterator() { @@ -132,7 +139,6 @@ protected NodeLabelRange fetchNextOrNull() fill( labelsForEachNode, null ); long nextLowestRange = Long.MAX_VALUE; - int slots = 0; try { // One "rangeSize" range at a time @@ -147,7 +153,9 @@ protected NodeLabelRange fetchNextOrNull() } else if ( idRange == currentRange ) { - slots = readRange( slots, cursor ); + long bits = cursor.get().value().bits; + long labelId = cursor.get().key().labelId; + readBitmap( bits, labelId, labelsForEachNode ); // Advance cursor and look ahead to the next range if ( cursor.next() ) @@ -162,7 +170,7 @@ else if ( idRange == currentRange ) } } - NativeNodeLabelRange range = new NativeNodeLabelRange( currentRange, convertState(), slots ); + NodeLabelRange range = new NodeLabelRange( currentRange, convertState( labelsForEachNode ) ); currentRange = nextLowestRange; return range; @@ -172,90 +180,5 @@ else if ( idRange == currentRange ) throw new RuntimeException( e ); } } - - private long[][] convertState() - { - long[][] labelIdsByNodeIndex = new long[RANGE_SIZE][]; - for ( int i = 0; i < RANGE_SIZE; i++ ) - { - List labelIdList = labelsForEachNode[i]; - if ( labelIdList != null ) - { - labelIdsByNodeIndex[i] = asArray( labelIdList.iterator() ); - } - } - return labelIdsByNodeIndex; - } - - private int readRange( int slots, RawCursor,IOException> cursor ) - { - long bits = cursor.get().value().bits; - while ( bits != 0 ) - { - int relativeNodeId = Long.numberOfTrailingZeros( bits ); - long labelId = cursor.get().key().labelId; - if ( labelsForEachNode[relativeNodeId] == null ) - { - labelsForEachNode[relativeNodeId] = new ArrayList<>(); - slots++; - } - labelsForEachNode[relativeNodeId].add( labelId ); - bits &= bits - 1; - } - return slots; - } - } - - private static class NativeNodeLabelRange extends NodeLabelRange - { - private final long idRange; - private final long[] nodes; - private final long[][] labels; - - NativeNodeLabelRange( long idRange, long[][] labels, int slots ) - { - this.idRange = idRange; - this.labels = labels; - long baseNodeId = idRange * RANGE_SIZE; - - this.nodes = new long[slots]; - int nodeIndex = 0; - for ( int i = 0; i < RANGE_SIZE; i++ ) - { - if ( labels[i] != null ) - { - nodes[nodeIndex++] = baseNodeId + i; - } - } - } - - @Override - public int id() - { - return (int) idRange; // TODO this is a weird thing, id and this conversion - } - - @Override - public long[] nodes() - { - return nodes; - } - - @Override - public long[] labels( long nodeId ) - { - long firstNodeId = idRange * RANGE_SIZE; - int index = toIntExact( nodeId - firstNodeId ); - assert index >= 0 && index < RANGE_SIZE : "nodeId:" + nodeId + ", idRange:" + idRange; - return labels[index]; - } - - @Override - public String toString() - { - String rangeString = idRange * RANGE_SIZE + "-" + (idRange + 1) * RANGE_SIZE; - String prefix = "NodeLabelRange[idRange=" + rangeString; - return toString( prefix, nodes, labels ); - } } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/impl/labelscan/LabelScanStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/api/impl/labelscan/LabelScanStoreTest.java index 29722b14da369..1baadf5ffe601 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/api/impl/labelscan/LabelScanStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/api/impl/labelscan/LabelScanStoreTest.java @@ -238,7 +238,7 @@ public void shouldScanSingleRange() throws Exception NodeLabelRange range = single( reader.iterator() ); // THEN - assertArrayEquals( new long[]{nodeId1, nodeId2}, sorted( range.nodes() ) ); + assertArrayEquals( new long[]{nodeId1, nodeId2}, reducedNodes( range ) ); assertArrayEquals( new long[]{labelId1}, sorted( range.labels( nodeId1 ) ) ); assertArrayEquals( new long[]{labelId1, labelId2}, sorted( range.labels( nodeId2 ) ) ); @@ -263,8 +263,8 @@ public void shouldScanMultipleRanges() throws Exception assertFalse( iterator.hasNext() ); // THEN - assertArrayEquals( new long[]{nodeId1}, sorted( range1.nodes() ) ); - assertArrayEquals( new long[]{nodeId2}, sorted( range2.nodes() ) ); + assertArrayEquals( new long[]{nodeId1}, reducedNodes( range1 ) ); + assertArrayEquals( new long[]{nodeId2}, reducedNodes( range2 ) ); assertArrayEquals( new long[]{labelId1}, sorted( range1.labels( nodeId1 ) ) ); @@ -331,7 +331,13 @@ public void shouldSeeEntriesWhenOnlyLowestIsPresent() throws Exception // when MutableInt count = new MutableInt(); AllEntriesLabelScanReader nodeLabelRanges = store.allNodeLabelRanges(); - nodeLabelRanges.forEach( nlr -> count.add( nlr.nodes().length ) ); + nodeLabelRanges.forEach( nlr -> + { + for ( long nodeId : nlr.nodes() ) + { + count.add( nlr.labels( nodeId ).length ); + } + } ); assertThat( count.intValue(), is( 1 ) ); } @@ -352,6 +358,21 @@ private long[] sorted( long[] input ) return input; } + private long[] reducedNodes( NodeLabelRange range ) + { + long[] nodes = range.nodes(); + long[] result = new long[nodes.length]; + int cursor = 0; + for ( long node : nodes ) + { + if ( range.labels( node ).length > 0 ) + { + result[cursor++] = node; + } + } + return Arrays.copyOf( result, cursor ); + } + @Test public void shouldRebuildFromScratchIfIndexMissing() throws Exception { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/labelscan/NodeLabelRangeTest.java b/community/kernel/src/test/java/org/neo4j/kernel/api/labelscan/NodeLabelRangeTest.java new file mode 100644 index 0000000000000..62c876d39797e --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/api/labelscan/NodeLabelRangeTest.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.api.labelscan; + +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; + +public class NodeLabelRangeTest +{ + @Test + public void shouldTransposeNodeIdsAndLabelIds() throws Exception + { + // given + long[][] labelsPerNode = new long[][] { + {1}, + {1, 3}, + {3, 5, 7}, + {}, + {1, 5, 7}, + {}, + {}, + {1, 2, 3, 4} + }; + + // when + NodeLabelRange range = new NodeLabelRange( 0, labelsPerNode ); + + // then + assertArrayEquals( new long[] {0, 1, 2, 3, 4, 5, 6, 7}, range.nodes() ); + for ( int i = 0; i < labelsPerNode.length; i++ ) + { + assertArrayEquals( labelsPerNode[i], range.labels( i ) ); + } + } + + @Test + public void shouldRebaseOnRangeId() throws Exception + { + // given + long[][] labelsPerNode = new long[][] { + {1}, + {1, 3}, + {3, 5, 7}, + {}, + {1, 5, 7}, + {}, + {}, + {1, 2, 3, 4} + }; + + // when + NodeLabelRange range = new NodeLabelRange( 10, labelsPerNode ); + + // then + long baseNodeId = range.id() * labelsPerNode.length; + long[] expectedNodeIds = new long[labelsPerNode.length]; + for ( int i = 0; i < expectedNodeIds.length; i++ ) + { + expectedNodeIds[i] = baseNodeId + i; + } + assertArrayEquals( expectedNodeIds, range.nodes() ); + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeAllEntriesLabelScanReaderTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeAllEntriesLabelScanReaderTest.java index e956f9883e69a..ff671bd10f32c 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeAllEntriesLabelScanReaderTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/labelscan/NativeAllEntriesLabelScanReaderTest.java @@ -141,7 +141,6 @@ private static void assertRanges( AllEntriesLabelScanReader reader, Labels[] dat NodeLabelRange range = iterator.next(); assertEquals( rangeId, range.id() ); - assertArrayEquals( "Unexpected data in range " + rangeId, nodesOf( expected ), range.nodes() ); for ( Map.Entry> expectedEntry : expected.entrySet() ) { long[] labels = range.labels( expectedEntry.getKey() ); @@ -153,17 +152,6 @@ private static void assertRanges( AllEntriesLabelScanReader reader, Labels[] dat assertFalse( iterator.hasNext() ); } - private static long[] nodesOf( SortedMap> expected ) - { - long[] nodes = new long[expected.size()]; - Iterator nodeIds = expected.keySet().iterator(); - for ( int i = 0; nodeIds.hasNext(); i++ ) - { - nodes[i] = nodeIds.next(); - } - return nodes; - } - private static SortedMap> rangeOf( Labels[] data, long rangeId ) { SortedMap> result = new TreeMap<>(); diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/labelscan/LuceneNodeLabelRange.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/labelscan/LuceneNodeLabelRange.java deleted file mode 100644 index 996af3fe433a1..0000000000000 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/labelscan/LuceneNodeLabelRange.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.api.impl.labelscan; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.neo4j.kernel.api.labelscan.NodeLabelRange; - - -public class LuceneNodeLabelRange extends NodeLabelRange -{ - private final int id; - private final long[] nodeIds; - private final long[][] labelIds; - - public LuceneNodeLabelRange( int id, long[] nodeIds, long[][] labelIds ) - { - this.id = id; - this.labelIds = labelIds; - this.nodeIds = nodeIds; - } - - @Override - public String toString() - { - String prefix = "NodeLabelRange[docId=" + id; - return toString( prefix, nodeIds, labelIds ); - } - - @Override - public int id() - { - return id; - } - - @Override - public long[] nodes() - { - return nodeIds; - } - - @Override - public long[] labels( long nodeId ) - { - for ( int i = 0; i < nodeIds.length; i++ ) - { - if ( nodeId == nodeIds[i] ) - { - return labelIds[i]; - } - } - throw new IllegalArgumentException( "Unknown nodeId: " + nodeId ); - } - - public static LuceneNodeLabelRange fromBitmapStructure( int id, long[] labelIds, long[][] nodeIdsByLabelIndex ) - { - Map> labelsForEachNode = new HashMap<>( ); - for ( int i = 0; i < labelIds.length; i++ ) - { - long labelId = labelIds[i]; - for ( int j = 0; j < nodeIdsByLabelIndex[i].length; j++ ) - { - long nodeId = nodeIdsByLabelIndex[i][j]; - List labelIdList = labelsForEachNode.get( nodeId ); - if ( labelIdList == null ) - { - labelsForEachNode.put( nodeId, labelIdList = new ArrayList<>() ); - } - labelIdList.add( labelId ); - } - } - - Set nodeIdSet = labelsForEachNode.keySet(); - long[] nodeIds = new long[ nodeIdSet.size() ]; - long[][] labelIdsByNodeIndex = new long[ nodeIdSet.size() ][]; - int nodeIndex = 0; - for ( long nodeId : nodeIdSet ) - { - nodeIds[ nodeIndex ] = nodeId; - List labelIdList = labelsForEachNode.get( nodeId ); - long[] nodeLabelIds = new long[ labelIdList.size() ]; - int labelIndex = 0; - for ( long labelId : labelIdList ) - { - nodeLabelIds[ labelIndex ] = labelId; - labelIndex++; - } - labelIdsByNodeIndex[ nodeIndex ] = nodeLabelIds; - nodeIndex++; - } - return new LuceneNodeLabelRange( id, nodeIds, labelIdsByNodeIndex ); - } -} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/labelscan/reader/LuceneAllEntriesLabelScanReader.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/labelscan/reader/LuceneAllEntriesLabelScanReader.java index 2ca1b281c75ff..200d0fc4e0577 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/labelscan/reader/LuceneAllEntriesLabelScanReader.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/labelscan/reader/LuceneAllEntriesLabelScanReader.java @@ -25,14 +25,15 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; - import org.neo4j.helpers.collection.BoundedIterable; -import org.neo4j.kernel.api.impl.labelscan.LuceneNodeLabelRange; +import org.neo4j.helpers.collection.IteratorWrapper; import org.neo4j.kernel.api.impl.labelscan.bitmaps.Bitmap; import org.neo4j.kernel.api.impl.labelscan.storestrategy.BitmapDocumentFormat; import org.neo4j.kernel.api.labelscan.AllEntriesLabelScanReader; import org.neo4j.kernel.api.labelscan.NodeLabelRange; +import static org.neo4j.kernel.api.labelscan.NodeLabelRange.readBitmap; + public class LuceneAllEntriesLabelScanReader implements AllEntriesLabelScanReader { private final BitmapDocumentFormat format; @@ -48,27 +49,22 @@ public LuceneAllEntriesLabelScanReader( BoundedIterable documents, Bit public Iterator iterator() { final Iterator iterator = documents.iterator(); - return new Iterator() + return new IteratorWrapper( iterator ) { - private int id = 0; - - public boolean hasNext() - { - return iterator.hasNext(); - } - - public LuceneNodeLabelRange next() - { - return parse( id++, iterator.next() ); - } - - public void remove() + @Override + protected NodeLabelRange underlyingObjectToObject( Document document ) { - iterator.remove(); + return parse( document ); } }; } + @Override + public int rangeSize() + { + return format.bitmapFormat().rangeSize(); + } + @Override public void close() throws Exception { @@ -81,7 +77,7 @@ public long maxCount() return documents.maxCount(); } - private LuceneNodeLabelRange parse( int id, Document document ) + private NodeLabelRange parse( Document document ) { List fields = document.getFields(); @@ -119,16 +115,12 @@ else if ( format.isLabelBitmapField( field ) ) bitmaps = scratchBitmaps; } - return LuceneNodeLabelRange.fromBitmapStructure( id, labelIds, getLongs( bitmaps, rangeId ) ); - } - - private long[][] getLongs( Bitmap[] bitmaps, long rangeId ) - { - long[][] nodeIds = new long[bitmaps.length][]; - for ( int k = 0; k < nodeIds.length; k++ ) + @SuppressWarnings( "unchecked" ) + List[] labelsPerNode = new List[format.bitmapFormat().rangeSize()]; + for ( int j = 0; j < labelIds.length; j++ ) { - nodeIds[k] = format.bitmapFormat().convertRangeAndBitmapToArray( rangeId, bitmaps[k].bitmap() ); + readBitmap( bitmaps[j].bitmap(), labelIds[j], labelsPerNode ); } - return nodeIds; + return new NodeLabelRange( rangeId, NodeLabelRange.convertState( labelsPerNode ) ); } } diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/labelscan/LuceneNodeLabelRangeTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/labelscan/LuceneNodeLabelRangeTest.java deleted file mode 100644 index ae277365f2028..0000000000000 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/labelscan/LuceneNodeLabelRangeTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.api.impl.labelscan; - -import org.junit.Test; - -import org.neo4j.helpers.collection.Iterables; - -import static org.hamcrest.CoreMatchers.hasItems; -import static org.junit.Assert.assertThat; - -public class LuceneNodeLabelRangeTest -{ - @Test - public void shouldTransposeNodeIdsAndLabelIds() throws Exception - { - // given - long[] labelIds = new long[]{1, 3, 5}; - long[][] nodeIdsByLabelIndex = new long[][]{{0}, {2, 4}, {0, 2, 4}}; - - // when - LuceneNodeLabelRange range = LuceneNodeLabelRange.fromBitmapStructure( 0, labelIds, nodeIdsByLabelIndex ); - - // then - assertThat( Iterables.asIterable( range.nodes() ), hasItems(0L, 2L, 4L)); - assertThat( Iterables.asIterable( range.labels( 0 ) ), hasItems( 1L, 5L ) ); - assertThat( Iterables.asIterable( range.labels( 2 ) ), hasItems( 3L, 5L ) ); - assertThat( Iterables.asIterable( range.labels( 4 ) ), hasItems( 3L, 5L ) ); - } -}