-
Notifications
You must be signed in to change notification settings - Fork 2.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Close all index readers opened during lockingNodeUniqueIndexSeek
Previously one of the readers was leaked and that forcing lucene index to keep snapshot forever till next restart. After X calls that where cause uncontrollable index size and the number of files grows that were causing all sort of problems: OOD, OOM, inability to calculate store size over JMX, etc.
- Loading branch information
1 parent
dd38b10
commit 84cb493
Showing
9 changed files
with
614 additions
and
35 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
56 changes: 56 additions & 0 deletions
56
community/kernel/src/main/java/org/neo4j/kernel/impl/newapi/IndexReaders.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| /* | ||
| * Copyright (c) 2002-2018 "Neo4j," | ||
| * Neo4j Sweden AB [http://neo4j.com] | ||
| * | ||
| * This file is part of Neo4j. | ||
| * | ||
| * Neo4j is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| package org.neo4j.kernel.impl.newapi; | ||
|
|
||
| import java.io.Closeable; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| import org.neo4j.internal.kernel.api.IndexReference; | ||
| import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException; | ||
| import org.neo4j.storageengine.api.schema.IndexReader; | ||
|
|
||
| import static org.neo4j.io.IOUtils.closeAllSilently; | ||
|
|
||
| class IndexReaders implements Closeable | ||
| { | ||
| private final List<IndexReader> indexReaders = new ArrayList<>(); | ||
| private final IndexReference indexReference; | ||
| private final Read read; | ||
|
|
||
| IndexReaders( IndexReference indexReference, Read read ) | ||
| { | ||
| this.indexReference = indexReference; | ||
| this.read = read; | ||
| } | ||
|
|
||
| IndexReader createReader() throws IndexNotFoundKernelException | ||
| { | ||
| IndexReader indexReader = read.indexReader( indexReference, true ); | ||
| indexReaders.add( indexReader ); | ||
| return indexReader; | ||
| } | ||
|
|
||
| @Override | ||
| public void close() | ||
| { | ||
| closeAllSilently( indexReaders ); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
137 changes: 137 additions & 0 deletions
137
community/neo4j/src/test/java/org/neo4j/kernel/impl/index/schema/UniqueIndexSeekIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| /* | ||
| * Copyright (c) 2002-2018 "Neo4j," | ||
| * Neo4j Sweden AB [http://neo4j.com] | ||
| * | ||
| * This file is part of Neo4j. | ||
| * | ||
| * Neo4j is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
| package org.neo4j.kernel.impl.index.schema; | ||
|
|
||
| import org.junit.Rule; | ||
| import org.junit.Test; | ||
|
|
||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| import org.neo4j.graphdb.Label; | ||
| import org.neo4j.graphdb.Node; | ||
| import org.neo4j.graphdb.Transaction; | ||
| import org.neo4j.internal.kernel.api.CapableIndexReference; | ||
| import org.neo4j.internal.kernel.api.IndexQuery; | ||
| import org.neo4j.internal.kernel.api.Read; | ||
| import org.neo4j.internal.kernel.api.TokenRead; | ||
| import org.neo4j.internal.kernel.api.exceptions.KernelException; | ||
| import org.neo4j.kernel.api.KernelTransaction; | ||
| import org.neo4j.kernel.impl.core.ThreadToStatementContextBridge; | ||
| import org.neo4j.kernel.impl.index.schema.tracking.TrackingIndexExtensionFactory; | ||
| import org.neo4j.kernel.internal.GraphDatabaseAPI; | ||
| import org.neo4j.test.TestGraphDatabaseFactory; | ||
| import org.neo4j.test.rule.TestDirectory; | ||
| import org.neo4j.test.rule.fs.DefaultFileSystemRule; | ||
|
|
||
| import static java.util.Collections.singletonList; | ||
| import static org.hamcrest.Matchers.greaterThan; | ||
| import static org.junit.Assert.assertEquals; | ||
| import static org.junit.Assert.assertNotNull; | ||
| import static org.junit.Assert.assertThat; | ||
| import static org.neo4j.graphdb.Label.label; | ||
| import static org.neo4j.kernel.impl.index.schema.tracking.TrackingReadersIndexAccessor.numberOfClosedReaders; | ||
| import static org.neo4j.kernel.impl.index.schema.tracking.TrackingReadersIndexAccessor.numberOfOpenReaders; | ||
|
|
||
| public class UniqueIndexSeekIT | ||
| { | ||
| @Rule | ||
| public final DefaultFileSystemRule fs = new DefaultFileSystemRule(); | ||
| @Rule | ||
| public final TestDirectory directory = TestDirectory.testDirectory( fs ); | ||
|
|
||
| @Test | ||
| public void uniqueIndexSeekDoNotLeakIndexReaders() throws KernelException | ||
| { | ||
| TrackingIndexExtensionFactory indexExtensionFactory = new TrackingIndexExtensionFactory(); | ||
| GraphDatabaseAPI database = createDatabase( indexExtensionFactory ); | ||
| try | ||
| { | ||
|
|
||
| Label label = label( "spaceship" ); | ||
| String nameProperty = "name"; | ||
| createUniqueConstraint( database, label, nameProperty ); | ||
|
|
||
| generateRandomData( database, label, nameProperty ); | ||
|
|
||
| assertNotNull( indexExtensionFactory.getIndexProvider() ); | ||
| assertThat( numberOfClosedReaders(), greaterThan( 0L ) ); | ||
| assertThat( numberOfOpenReaders(), greaterThan( 0L ) ); | ||
| assertEquals( numberOfClosedReaders(), numberOfOpenReaders() ); | ||
|
|
||
| lockNodeUsingUniqueIndexSeek( database, label, nameProperty ); | ||
|
|
||
| assertEquals( numberOfClosedReaders(), numberOfOpenReaders() ); | ||
| } | ||
| finally | ||
| { | ||
| database.shutdown(); | ||
| } | ||
| } | ||
|
|
||
| private GraphDatabaseAPI createDatabase( TrackingIndexExtensionFactory indexExtensionFactory ) | ||
| { | ||
| return (GraphDatabaseAPI) new TestGraphDatabaseFactory() | ||
| .setKernelExtensions( singletonList( indexExtensionFactory ) ).newEmbeddedDatabaseBuilder( directory.graphDbDir() ).newGraphDatabase(); | ||
| } | ||
|
|
||
| private void lockNodeUsingUniqueIndexSeek( GraphDatabaseAPI database, Label label, String nameProperty ) throws KernelException | ||
| { | ||
| try ( Transaction transaction = database.beginTx() ) | ||
| { | ||
| ThreadToStatementContextBridge contextBridge = database.getDependencyResolver().resolveDependency( ThreadToStatementContextBridge.class ); | ||
| KernelTransaction kernelTransaction = contextBridge.getKernelTransactionBoundToThisThread( true ); | ||
| TokenRead tokenRead = kernelTransaction.tokenRead(); | ||
| Read dataRead = kernelTransaction.dataRead(); | ||
|
|
||
| int labelId = tokenRead.nodeLabel( label.name() ); | ||
| int propertyId = tokenRead.propertyKey( nameProperty ); | ||
| CapableIndexReference indexReference = kernelTransaction.schemaRead().index( labelId, propertyId ); | ||
| dataRead.lockingNodeUniqueIndexSeek( indexReference, IndexQuery.ExactPredicate.exact( propertyId, "value" ) ); | ||
| transaction.success(); | ||
| } | ||
| } | ||
|
|
||
| private void generateRandomData( GraphDatabaseAPI database, Label label, String nameProperty ) | ||
| { | ||
| for ( int i = 0; i < 1000; i++ ) | ||
| { | ||
| try ( Transaction transaction = database.beginTx() ) | ||
| { | ||
| Node node = database.createNode( label ); | ||
| node.setProperty( nameProperty, "PlanetExpress" + i ); | ||
| transaction.success(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private void createUniqueConstraint( GraphDatabaseAPI database, Label label, String nameProperty ) | ||
| { | ||
| try ( Transaction transaction = database.beginTx() ) | ||
| { | ||
| database.schema().constraintFor( label ).assertPropertyIsUnique( nameProperty ).create(); | ||
| transaction.success(); | ||
| } | ||
| try ( Transaction transaction = database.beginTx() ) | ||
| { | ||
| database.schema().awaitIndexesOnline( 1, TimeUnit.MINUTES ); | ||
| transaction.success(); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.